* [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
@ 2018-08-15 9:48 Emily Deng
[not found] ` <1534326508-17477-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Emily Deng @ 2018-08-15 9:48 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: Emily Deng
To avoid the tlb flush not interrupted by world switch, use kiq and one
command to do tlb invalidate.
v2:
Add firmware version checking.
v3:
Refine the code, and move the firmware
checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
SWDEV-161497
Signed-off-by: Emily Deng <Emily.Deng@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61 ++++++++++++++++++++++++++++++++
4 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 07924d4..67b584b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
AMDGPU_CP_KIQ_IRQ_LAST
};
+#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
+#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
+#define MAX_KIQ_REG_TRY 20
+
int amdgpu_device_ip_set_clockgating_state(void *dev,
enum amd_ip_block_type block_type,
enum amd_clockgating_state state);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 21adb1b6..3885636 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -22,9 +22,6 @@
*/
#include "amdgpu.h"
-#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
-#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
-#define MAX_KIQ_REG_TRY 20
uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 76d979e..c9b3db4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4348,8 +4348,21 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
uint32_t ref, uint32_t mask)
{
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+ struct amdgpu_device *adev = ring->adev;
+ bool fw_version_ok = false;
- if (amdgpu_sriov_vf(ring->adev))
+ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+ if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev->gfx.me_feature_version >= 42))
+ if ((adev->gfx.pfp_fw_version >= 0x000000b1) && (adev->gfx.pfp_feature_version >= 42))
+ fw_version_ok = true;
+ } else {
+ if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev->gfx.mec_feature_version >= 42))
+ fw_version_ok = true;
+ }
+
+ fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok : false;
+
+ if (amdgpu_sriov_vf(adev) && fw_version_ok)
gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
ref, mask, 0x20);
else
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..3419178 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -311,6 +311,60 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
return req;
}
+signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+ unsigned eng, u32 req, uint32_t vmid)
+{
+ signed long r, cnt = 0;
+ unsigned long flags;
+ uint32_t seq;
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *ring = &kiq->ring;
+
+ if (!ring->ready) {
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&kiq->ring_lock, flags);
+
+ amdgpu_ring_alloc(ring, 32);
+ amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+ hub->vm_inv_eng0_ack + eng,
+ req, 1 << vmid);
+ amdgpu_fence_emit_polling(ring, &seq);
+ amdgpu_ring_commit(ring);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+ /* don't wait anymore for gpu reset case because this way may
+ * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+ * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+ * never return if we keep waiting in virt_kiq_rreg, which cause
+ * gpu_recover() hang there.
+ *
+ * also don't wait anymore for IRQ context
+ * */
+ if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+ goto failed_kiq;
+
+ if (in_interrupt())
+ might_sleep();
+
+ while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+ msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+ }
+
+ if (cnt > MAX_KIQ_REG_TRY)
+ goto failed_kiq;
+
+ return 0;
+
+failed_kiq:
+ pr_err("failed to invalidate tlb with kiq\n");
+ return r;
+}
+
/*
* GART
* VMID 0 is the physical GPU addresses as used by the kernel.
@@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
/* Use register 17 for GART */
const unsigned eng = 17;
unsigned i, j;
+ int r;
spin_lock(&adev->gmc.invalidate_lock);
@@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
struct amdgpu_vmhub *hub = &adev->vmhub[i];
u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
+ if (amdgpu_sriov_vf(adev) ) {
+ r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+ if (!r)
+ continue;
+ }
+
WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
/* Busy wait for ACK.*/
--
2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 15+ messages in thread
* [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register.
[not found] ` <1534326508-17477-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
@ 2018-08-15 9:48 ` Emily Deng
[not found] ` <1534326508-17477-2-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 10:50 ` [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb Christian König
2018-08-15 13:56 ` Zhu, Rex
2 siblings, 1 reply; 15+ messages in thread
From: Emily Deng @ 2018-08-15 9:48 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: Emily Deng
Signed-off-by: Emily Deng <Emily.Deng@amd.com>
---
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index b60ed28..52b13e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -693,7 +693,10 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
amdgpu_device_vram_location(adev, &adev->gmc, base);
amdgpu_device_gart_location(adev, mc);
/* base offset of vram pages */
- adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
+ if (!amdgpu_sriov_vf(adev))
+ adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
+ else
+ adev->vm_manager.vram_base_offset = 0;
}
/**
--
2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 15+ messages in thread
* RE: [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register.
[not found] ` <1534326508-17477-2-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
@ 2018-08-15 9:52 ` Deng, Emily
0 siblings, 0 replies; 15+ messages in thread
From: Deng, Emily @ 2018-08-15 9:52 UTC (permalink / raw)
To: Deng, Emily, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
Sorry, sent the wrong patch, please ignore this email.
>-----Original Message-----
>From: Emily Deng <Emily.Deng@amd.com>
>Sent: Wednesday, August 15, 2018 5:48 PM
>To: amd-gfx@lists.freedesktop.org
>Cc: Deng, Emily <Emily.Deng@amd.com>
>Subject: [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register.
>
>Signed-off-by: Emily Deng <Emily.Deng@amd.com>
>---
> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 5 ++++-
> 1 file changed, 4 insertions(+), 1 deletion(-)
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>index b60ed28..52b13e6 100644
>--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>@@ -693,7 +693,10 @@ static void gmc_v9_0_vram_gtt_location(struct
>amdgpu_device *adev,
> amdgpu_device_vram_location(adev, &adev->gmc, base);
> amdgpu_device_gart_location(adev, mc);
> /* base offset of vram pages */
>- adev->vm_manager.vram_base_offset =
>gfxhub_v1_0_get_mc_fb_offset(adev);
>+ if (!amdgpu_sriov_vf(adev))
>+ adev->vm_manager.vram_base_offset =
>gfxhub_v1_0_get_mc_fb_offset(adev);
>+ else
>+ adev->vm_manager.vram_base_offset = 0;
> }
>
> /**
>--
>2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <1534326508-17477-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 9:48 ` [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register Emily Deng
@ 2018-08-15 10:50 ` Christian König
[not found] ` <5f51ffa1-1e84-dacf-840c-76bd9cbb282b-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
2018-08-15 13:56 ` Zhu, Rex
2 siblings, 1 reply; 15+ messages in thread
From: Christian König @ 2018-08-15 10:50 UTC (permalink / raw)
To: Emily Deng, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
Am 15.08.2018 um 11:48 schrieb Emily Deng:
> To avoid the tlb flush not interrupted by world switch, use kiq and one
> command to do tlb invalidate.
>
> v2:
> Add firmware version checking.
>
> v3:
> Refine the code, and move the firmware
> checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
>
> SWDEV-161497
>
> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61 ++++++++++++++++++++++++++++++++
> 4 files changed, 79 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 07924d4..67b584b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
> AMDGPU_CP_KIQ_IRQ_LAST
> };
>
> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> +#define MAX_KIQ_REG_TRY 20
> +
> int amdgpu_device_ip_set_clockgating_state(void *dev,
> enum amd_ip_block_type block_type,
> enum amd_clockgating_state state);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 21adb1b6..3885636 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -22,9 +22,6 @@
> */
>
> #include "amdgpu.h"
> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> -#define MAX_KIQ_REG_TRY 20
>
> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
> {
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 76d979e..c9b3db4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -4348,8 +4348,21 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> uint32_t ref, uint32_t mask)
> {
> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> + struct amdgpu_device *adev = ring->adev;
> + bool fw_version_ok = false;
>
> - if (amdgpu_sriov_vf(ring->adev))
> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev->gfx.me_feature_version >= 42))
> + if ((adev->gfx.pfp_fw_version >= 0x000000b1) && (adev->gfx.pfp_feature_version >= 42))
Why "if (...) if (...)" ? Just if (... && ...) should do as well.
> + fw_version_ok = true;
> + } else {
> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev->gfx.mec_feature_version >= 42))
> + fw_version_ok = true;
> + }
> +
> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok : false;
> +
> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> ref, mask, 0x20);
> else
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index ed467de..3419178 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -311,6 +311,60 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
> return req;
> }
>
> +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
> + unsigned eng, u32 req, uint32_t vmid)
Better make that function just a generic reg_write_reg_wait function,
instead of specialized for tlb flushing.
> +{
> + signed long r, cnt = 0;
> + unsigned long flags;
> + uint32_t seq;
> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> + struct amdgpu_ring *ring = &kiq->ring;
> +
> + if (!ring->ready) {
> + return -EINVAL;
> + }
> +
> + spin_lock_irqsave(&kiq->ring_lock, flags);
> +
> + amdgpu_ring_alloc(ring, 32);
> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
> + hub->vm_inv_eng0_ack + eng,
> + req, 1 << vmid);
> + amdgpu_fence_emit_polling(ring, &seq);
> + amdgpu_ring_commit(ring);
> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> +
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> +
> + /* don't wait anymore for gpu reset case because this way may
> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> + * never return if we keep waiting in virt_kiq_rreg, which cause
> + * gpu_recover() hang there.
> + *
> + * also don't wait anymore for IRQ context
> + * */
> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> + goto failed_kiq;
> +
> + if (in_interrupt())
> + might_sleep();
What should that be good for?
> +
> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> + }
> +
> + if (cnt > MAX_KIQ_REG_TRY)
> + goto failed_kiq;
> +
> + return 0;
> +
> +failed_kiq:
> + pr_err("failed to invalidate tlb with kiq\n");
> + return r;
> +}
> +
> /*
> * GART
> * VMID 0 is the physical GPU addresses as used by the kernel.
> @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
> /* Use register 17 for GART */
> const unsigned eng = 17;
> unsigned i, j;
> + int r;
>
> spin_lock(&adev->gmc.invalidate_lock);
>
> @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
> struct amdgpu_vmhub *hub = &adev->vmhub[i];
> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>
> + if (amdgpu_sriov_vf(adev) ) {
As discussed drop the sriov check here and use the KIQ path all the time.
Christian.
> + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
> + if (!r)
> + continue;
> + }
> +
> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>
> /* Busy wait for ACK.*/
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <5f51ffa1-1e84-dacf-840c-76bd9cbb282b-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
@ 2018-08-15 11:08 ` Deng, Emily
[not found] ` <CY4PR12MB112541E31A354E9E94BE2FCF8F3F0-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Deng, Emily @ 2018-08-15 11:08 UTC (permalink / raw)
To: Koenig, Christian, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
>-----Original Message-----
>From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
>Christian König
>Sent: Wednesday, August 15, 2018 6:50 PM
>To: Deng, Emily <Emily.Deng@amd.com>; amd-gfx@lists.freedesktop.org
>Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
>invalidate tlb
>
>Am 15.08.2018 um 11:48 schrieb Emily Deng:
>> To avoid the tlb flush not interrupted by world switch, use kiq and
>> one command to do tlb invalidate.
>>
>> v2:
>> Add firmware version checking.
>>
>> v3:
>> Refine the code, and move the firmware checking into
>> gfx_v9_0_ring_emit_reg_write_reg_wait.
>>
>> SWDEV-161497
>>
>> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
>> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
>> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
>++++++++++++++++++++++++++++++++
>> 4 files changed, 79 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> index 07924d4..67b584b 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
>> AMDGPU_CP_KIQ_IRQ_LAST
>> };
>>
>> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>> +#define MAX_KIQ_REG_TRY 20
>> +
>> int amdgpu_device_ip_set_clockgating_state(void *dev,
>> enum amd_ip_block_type block_type,
>> enum amd_clockgating_state state);
>diff --git
>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> index 21adb1b6..3885636 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> @@ -22,9 +22,6 @@
>> */
>>
>> #include "amdgpu.h"
>> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>> -#define MAX_KIQ_REG_TRY 20
>>
>> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
>> {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> index 76d979e..c9b3db4 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> @@ -4348,8 +4348,21 @@ static void
>gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
>> uint32_t ref, uint32_t mask)
>> {
>> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
>> + struct amdgpu_device *adev = ring->adev;
>> + bool fw_version_ok = false;
>>
>> - if (amdgpu_sriov_vf(ring->adev))
>> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
>> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
>>gfx.me_feature_version >= 42))
>> + if ((adev->gfx.pfp_fw_version >= 0x000000b1) &&
>> +(adev->gfx.pfp_feature_version >= 42))
>
>Why "if (...) if (...)" ? Just if (... && ...) should do as well.
Ok, will modify.
>
>> + fw_version_ok = true;
>> + } else {
>> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev-
>>gfx.mec_feature_version >= 42))
>> + fw_version_ok = true;
>> + }
>> +
>> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok :
>> +false;
>> +
>> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
>> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
>> ref, mask, 0x20);
>> else
>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> index ed467de..3419178 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> @@ -311,6 +311,60 @@ static uint32_t
>gmc_v9_0_get_invalidate_req(unsigned int vmid)
>> return req;
>> }
>>
>> +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct
>amdgpu_vmhub *hub,
>> + unsigned eng, u32 req, uint32_t vmid)
>
>Better make that function just a generic reg_write_reg_wait function, instead of
>specialized for tlb flushing.
Do you mean rename the function to amdgpu_reg_write_reg_wait?
>
>> +{
>> + signed long r, cnt = 0;
>> + unsigned long flags;
>> + uint32_t seq;
>> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> + struct amdgpu_ring *ring = &kiq->ring;
>> +
>> + if (!ring->ready) {
>> + return -EINVAL;
>> + }
>> +
>> + spin_lock_irqsave(&kiq->ring_lock, flags);
>> +
>> + amdgpu_ring_alloc(ring, 32);
>> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
>eng,
>> + hub->vm_inv_eng0_ack + eng,
>> + req, 1 << vmid);
>> + amdgpu_fence_emit_polling(ring, &seq);
>> + amdgpu_ring_commit(ring);
>> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
>> +
>> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
>> +
>> + /* don't wait anymore for gpu reset case because this way may
>> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
>> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
>> + * never return if we keep waiting in virt_kiq_rreg, which cause
>> + * gpu_recover() hang there.
>> + *
>> + * also don't wait anymore for IRQ context
>> + * */
>> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
>> + goto failed_kiq;
>> +
>> + if (in_interrupt())
>> + might_sleep();
>
>What should that be good for?
It is only used for the interrupt case.
>> +
>> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
>> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
>> + r = amdgpu_fence_wait_polling(ring, seq,
>MAX_KIQ_REG_WAIT);
>> + }
>> +
>> + if (cnt > MAX_KIQ_REG_TRY)
>> + goto failed_kiq;
>> +
>> + return 0;
>> +
>> +failed_kiq:
>> + pr_err("failed to invalidate tlb with kiq\n");
>> + return r;
>> +}
>> +
>> /*
>> * GART
>> * VMID 0 is the physical GPU addresses as used by the kernel.
>> @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>amdgpu_device *adev,
>> /* Use register 17 for GART */
>> const unsigned eng = 17;
>> unsigned i, j;
>> + int r;
>>
>> spin_lock(&adev->gmc.invalidate_lock);
>>
>> @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>amdgpu_device *adev,
>> struct amdgpu_vmhub *hub = &adev->vmhub[i];
>> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>>
>> + if (amdgpu_sriov_vf(adev) ) {
>
>As discussed drop the sriov check here and use the KIQ path all the time.
Already drop it in the patch 2.
Best wishes
Emily Deng
>Christian.
>
>> + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp,
>vmid);
>> + if (!r)
>> + continue;
>> + }
>> +
>> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>>
>> /* Busy wait for ACK.*/
>
>_______________________________________________
>amd-gfx mailing list
>amd-gfx@lists.freedesktop.org
>https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <CY4PR12MB112541E31A354E9E94BE2FCF8F3F0-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
@ 2018-08-15 11:14 ` Christian König
[not found] ` <8a5bff61-c03f-856a-3f0e-3497b2bd0ba8-5C7GfCeVMHo@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Christian König @ 2018-08-15 11:14 UTC (permalink / raw)
To: Deng, Emily, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
Am 15.08.2018 um 13:08 schrieb Deng, Emily:
>> -----Original Message-----
>> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
>> Christian König
>> Sent: Wednesday, August 15, 2018 6:50 PM
>> To: Deng, Emily <Emily.Deng@amd.com>; amd-gfx@lists.freedesktop.org
>> Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
>> invalidate tlb
>>
>> Am 15.08.2018 um 11:48 schrieb Emily Deng:
>>> To avoid the tlb flush not interrupted by world switch, use kiq and
>>> one command to do tlb invalidate.
>>>
>>> v2:
>>> Add firmware version checking.
>>>
>>> v3:
>>> Refine the code, and move the firmware checking into
>>> gfx_v9_0_ring_emit_reg_write_reg_wait.
>>>
>>> SWDEV-161497
>>>
>>> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
>>> ---
>>> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
>>> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
>>> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
>> ++++++++++++++++++++++++++++++++
>>> 4 files changed, 79 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> index 07924d4..67b584b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
>>> AMDGPU_CP_KIQ_IRQ_LAST
>>> };
>>>
>>> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>>> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>>> +#define MAX_KIQ_REG_TRY 20
>>> +
>>> int amdgpu_device_ip_set_clockgating_state(void *dev,
>>> enum amd_ip_block_type block_type,
>>> enum amd_clockgating_state state);
>> diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> index 21adb1b6..3885636 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -22,9 +22,6 @@
>>> */
>>>
>>> #include "amdgpu.h"
>>> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>>> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>>> -#define MAX_KIQ_REG_TRY 20
>>>
>>> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
>>> {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>> b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>> index 76d979e..c9b3db4 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>> @@ -4348,8 +4348,21 @@ static void
>> gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
>>> uint32_t ref, uint32_t mask)
>>> {
>>> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
>>> + struct amdgpu_device *adev = ring->adev;
>>> + bool fw_version_ok = false;
>>>
>>> - if (amdgpu_sriov_vf(ring->adev))
>>> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
>>> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
>>> gfx.me_feature_version >= 42))
>>> + if ((adev->gfx.pfp_fw_version >= 0x000000b1) &&
>>> +(adev->gfx.pfp_feature_version >= 42))
>> Why "if (...) if (...)" ? Just if (... && ...) should do as well.
> Ok, will modify.
>>> + fw_version_ok = true;
>>> + } else {
>>> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev-
>>> gfx.mec_feature_version >= 42))
>>> + fw_version_ok = true;
>>> + }
>>> +
>>> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok :
>>> +false;
>>> +
>>> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
>>> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
>>> ref, mask, 0x20);
>>> else
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>> index ed467de..3419178 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>> @@ -311,6 +311,60 @@ static uint32_t
>> gmc_v9_0_get_invalidate_req(unsigned int vmid)
>>> return req;
>>> }
>>>
>>> +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct
>> amdgpu_vmhub *hub,
>>> + unsigned eng, u32 req, uint32_t vmid)
>> Better make that function just a generic reg_write_reg_wait function, instead of
>> specialized for tlb flushing.
> Do you mean rename the function to amdgpu_reg_write_reg_wait?
Yeah, something like that. I suggest amdgpu_kiq_reg_write_reg_wait() or
otherwise you might run into a name clash.
>>> +{
>>> + signed long r, cnt = 0;
>>> + unsigned long flags;
>>> + uint32_t seq;
>>> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> + struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> + if (!ring->ready) {
>>> + return -EINVAL;
>>> + }
>>> +
>>> + spin_lock_irqsave(&kiq->ring_lock, flags);
>>> +
>>> + amdgpu_ring_alloc(ring, 32);
>>> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
>> eng,
>>> + hub->vm_inv_eng0_ack + eng,
>>> + req, 1 << vmid);
>>> + amdgpu_fence_emit_polling(ring, &seq);
>>> + amdgpu_ring_commit(ring);
>>> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
>>> +
>>> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
>>> +
>>> + /* don't wait anymore for gpu reset case because this way may
>>> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
>>> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
>>> + * never return if we keep waiting in virt_kiq_rreg, which cause
>>> + * gpu_recover() hang there.
>>> + *
>>> + * also don't wait anymore for IRQ context
>>> + * */
>>> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
>>> + goto failed_kiq;
>>> +
>>> + if (in_interrupt())
>>> + might_sleep();
>> What should that be good for?
> It is only used for the interrupt case.
Well, might_sleep() is a debugging helper which bails out if called in a
situation where you can't sleep.
Protecting that with "if (in_interrupt())" doesn't make any sense at
all. So what exactly is your intention here?
Thanks,
Christian.
>>> +
>>> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
>>> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
>>> + r = amdgpu_fence_wait_polling(ring, seq,
>> MAX_KIQ_REG_WAIT);
>>> + }
>>> +
>>> + if (cnt > MAX_KIQ_REG_TRY)
>>> + goto failed_kiq;
>>> +
>>> + return 0;
>>> +
>>> +failed_kiq:
>>> + pr_err("failed to invalidate tlb with kiq\n");
>>> + return r;
>>> +}
>>> +
>>> /*
>>> * GART
>>> * VMID 0 is the physical GPU addresses as used by the kernel.
>>> @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>> amdgpu_device *adev,
>>> /* Use register 17 for GART */
>>> const unsigned eng = 17;
>>> unsigned i, j;
>>> + int r;
>>>
>>> spin_lock(&adev->gmc.invalidate_lock);
>>>
>>> @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>> amdgpu_device *adev,
>>> struct amdgpu_vmhub *hub = &adev->vmhub[i];
>>> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>>>
>>> + if (amdgpu_sriov_vf(adev) ) {
>> As discussed drop the sriov check here and use the KIQ path all the time.
> Already drop it in the patch 2.
>
> Best wishes
> Emily Deng
>> Christian.
>>
>>> + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp,
>> vmid);
>>> + if (!r)
>>> + continue;
>>> + }
>>> +
>>> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>>>
>>> /* Busy wait for ACK.*/
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <8a5bff61-c03f-856a-3f0e-3497b2bd0ba8-5C7GfCeVMHo@public.gmane.org>
@ 2018-08-15 11:24 ` Deng, Emily
0 siblings, 0 replies; 15+ messages in thread
From: Deng, Emily @ 2018-08-15 11:24 UTC (permalink / raw)
To: Koenig, Christian, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
>-----Original Message-----
>From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
>Christian König
>Sent: Wednesday, August 15, 2018 7:14 PM
>To: Deng, Emily <Emily.Deng@amd.com>; amd-gfx@lists.freedesktop.org
>Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
>invalidate tlb
>
>Am 15.08.2018 um 13:08 schrieb Deng, Emily:
>>> -----Original Message-----
>>> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
>>> Christian König
>>> Sent: Wednesday, August 15, 2018 6:50 PM
>>> To: Deng, Emily <Emily.Deng@amd.com>; amd-gfx@lists.freedesktop.org
>>> Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq
>>> to do invalidate tlb
>>>
>>> Am 15.08.2018 um 11:48 schrieb Emily Deng:
>>>> To avoid the tlb flush not interrupted by world switch, use kiq and
>>>> one command to do tlb invalidate.
>>>>
>>>> v2:
>>>> Add firmware version checking.
>>>>
>>>> v3:
>>>> Refine the code, and move the firmware checking into
>>>> gfx_v9_0_ring_emit_reg_write_reg_wait.
>>>>
>>>> SWDEV-161497
>>>>
>>>> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
>>>> ---
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
>>>> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
>>>> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
>>> ++++++++++++++++++++++++++++++++
>>>> 4 files changed, 79 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> index 07924d4..67b584b 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>> @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
>>>> AMDGPU_CP_KIQ_IRQ_LAST
>>>> };
>>>>
>>>> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>>>> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>>>> +#define MAX_KIQ_REG_TRY 20
>>>> +
>>>> int amdgpu_device_ip_set_clockgating_state(void *dev,
>>>> enum amd_ip_block_type block_type,
>>>> enum amd_clockgating_state state);
>>> diff --git
>>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>> index 21adb1b6..3885636 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>> @@ -22,9 +22,6 @@
>>>> */
>>>>
>>>> #include "amdgpu.h"
>>>> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>>>> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>>>> -#define MAX_KIQ_REG_TRY 20
>>>>
>>>> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
>>>> {
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>>> index 76d979e..c9b3db4 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>>>> @@ -4348,8 +4348,21 @@ static void
>>> gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
>>>> uint32_t ref, uint32_t mask)
>>>> {
>>>> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
>>>> + struct amdgpu_device *adev = ring->adev;
>>>> + bool fw_version_ok = false;
>>>>
>>>> - if (amdgpu_sriov_vf(ring->adev))
>>>> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
>>>> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
>>>> gfx.me_feature_version >= 42))
>>>> + if ((adev->gfx.pfp_fw_version >= 0x000000b1) &&
>>>> +(adev->gfx.pfp_feature_version >= 42))
>>> Why "if (...) if (...)" ? Just if (... && ...) should do as well.
>> Ok, will modify.
>>>> + fw_version_ok = true;
>>>> + } else {
>>>> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev-
>>>> gfx.mec_feature_version >= 42))
>>>> + fw_version_ok = true;
>>>> + }
>>>> +
>>>> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok :
>>>> +false;
>>>> +
>>>> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
>>>> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
>>>> ref, mask, 0x20);
>>>> else
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> index ed467de..3419178 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>>>> @@ -311,6 +311,60 @@ static uint32_t
>>> gmc_v9_0_get_invalidate_req(unsigned int vmid)
>>>> return req;
>>>> }
>>>>
>>>> +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev,
>>>> +struct
>>> amdgpu_vmhub *hub,
>>>> + unsigned eng, u32 req, uint32_t vmid)
>>> Better make that function just a generic reg_write_reg_wait function,
>>> instead of specialized for tlb flushing.
>> Do you mean rename the function to amdgpu_reg_write_reg_wait?
>
>Yeah, something like that. I suggest amdgpu_kiq_reg_write_reg_wait() or
>otherwise you might run into a name clash.
>
>>>> +{
>>>> + signed long r, cnt = 0;
>>>> + unsigned long flags;
>>>> + uint32_t seq;
>>>> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>>> + struct amdgpu_ring *ring = &kiq->ring;
>>>> +
>>>> + if (!ring->ready) {
>>>> + return -EINVAL;
>>>> + }
>>>> +
>>>> + spin_lock_irqsave(&kiq->ring_lock, flags);
>>>> +
>>>> + amdgpu_ring_alloc(ring, 32);
>>>> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
>>> eng,
>>>> + hub->vm_inv_eng0_ack + eng,
>>>> + req, 1 << vmid);
>>>> + amdgpu_fence_emit_polling(ring, &seq);
>>>> + amdgpu_ring_commit(ring);
>>>> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
>>>> +
>>>> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
>>>> +
>>>> + /* don't wait anymore for gpu reset case because this way may
>>>> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
>>>> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
>>>> + * never return if we keep waiting in virt_kiq_rreg, which cause
>>>> + * gpu_recover() hang there.
>>>> + *
>>>> + * also don't wait anymore for IRQ context
>>>> + * */
>>>> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
>>>> + goto failed_kiq;
>>>> +
>>>> + if (in_interrupt())
>>>> + might_sleep();
>>> What should that be good for?
>> It is only used for the interrupt case.
>
>Well, might_sleep() is a debugging helper which bails out if called in a situation
>where you can't sleep.
>
>Protecting that with "if (in_interrupt())" doesn't make any sense at all. So what
>exactly is your intention here?
Understand, will remove in_interrupt().
>
>Thanks,
>Christian.
>
>>>> +
>>>> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
>>>> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
>>>> + r = amdgpu_fence_wait_polling(ring, seq,
>>> MAX_KIQ_REG_WAIT);
>>>> + }
>>>> +
>>>> + if (cnt > MAX_KIQ_REG_TRY)
>>>> + goto failed_kiq;
>>>> +
>>>> + return 0;
>>>> +
>>>> +failed_kiq:
>>>> + pr_err("failed to invalidate tlb with kiq\n");
>>>> + return r;
>>>> +}
>>>> +
>>>> /*
>>>> * GART
>>>> * VMID 0 is the physical GPU addresses as used by the kernel.
>>>> @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>>> amdgpu_device *adev,
>>>> /* Use register 17 for GART */
>>>> const unsigned eng = 17;
>>>> unsigned i, j;
>>>> + int r;
>>>>
>>>> spin_lock(&adev->gmc.invalidate_lock);
>>>>
>>>> @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>>> amdgpu_device *adev,
>>>> struct amdgpu_vmhub *hub = &adev->vmhub[i];
>>>> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>>>>
>>>> + if (amdgpu_sriov_vf(adev) ) {
>>> As discussed drop the sriov check here and use the KIQ path all the time.
>> Already drop it in the patch 2.
>>
>> Best wishes
>> Emily Deng
>>> Christian.
>>>
>>>> + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp,
>>> vmid);
>>>> + if (!r)
>>>> + continue;
>>>> + }
>>>> +
>>>> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>>>>
>>>> /* Busy wait for ACK.*/
>>> _______________________________________________
>>> amd-gfx mailing list
>>> amd-gfx@lists.freedesktop.org
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
>_______________________________________________
>amd-gfx mailing list
>amd-gfx@lists.freedesktop.org
>https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <1534326508-17477-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 9:48 ` [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register Emily Deng
2018-08-15 10:50 ` [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb Christian König
@ 2018-08-15 13:56 ` Zhu, Rex
[not found] ` <BYAPR12MB27755A475CE994215D964F04FB3F0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
2 siblings, 1 reply; 15+ messages in thread
From: Zhu, Rex @ 2018-08-15 13:56 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: Deng, Emily
> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Emily
> Deng
> Sent: Wednesday, August 15, 2018 5:48 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Deng, Emily <Emily.Deng@amd.com>
> Subject: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
> invalidate tlb
>
> To avoid the tlb flush not interrupted by world switch, use kiq and one
> command to do tlb invalidate.
>
> v2:
> Add firmware version checking.
>
> v3:
> Refine the code, and move the firmware
> checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
>
> SWDEV-161497
The "SWDEV-161497" is meanless.
you can describe the issue or just remove the bug number.
>
> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
> ++++++++++++++++++++++++++++++++
> 4 files changed, 79 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 07924d4..67b584b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
> AMDGPU_CP_KIQ_IRQ_LAST
> };
>
> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> +#define MAX_KIQ_REG_TRY 20
> +
> int amdgpu_device_ip_set_clockgating_state(void *dev,
> enum amd_ip_block_type
> block_type,
> enum amd_clockgating_state state);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 21adb1b6..3885636 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -22,9 +22,6 @@
> */
>
> #include "amdgpu.h"
> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> -#define MAX_KIQ_REG_TRY 20
>
> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) { diff --git
> a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 76d979e..c9b3db4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -4348,8 +4348,21 @@ static void
> gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> uint32_t ref, uint32_t mask)
> {
> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> + struct amdgpu_device *adev = ring->adev;
> + bool fw_version_ok = false;
>
> - if (amdgpu_sriov_vf(ring->adev))
> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
> >gfx.me_feature_version >= 42))
> + if ((adev->gfx.pfp_fw_version >= 0x000000b1) &&
> (adev->gfx.pfp_feature_version >= 42))
> + fw_version_ok = true;
> + } else {
> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev-
> >gfx.mec_feature_version >= 42))
> + fw_version_ok = true;
> + }
Maybe we can add a flag and set the flag when request_firmware.
> +
> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok
> :
> +false;
> +
> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> ref, mask, 0x20);
> else
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index ed467de..3419178 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -311,6 +311,60 @@ static uint32_t
> gmc_v9_0_get_invalidate_req(unsigned int vmid)
> return req;
> }
>
> +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct
> amdgpu_vmhub *hub,
> + unsigned eng, u32 req, uint32_t vmid) {
> + signed long r, cnt = 0;
> + unsigned long flags;
> + uint32_t seq;
> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> + struct amdgpu_ring *ring = &kiq->ring;
> +
> + if (!ring->ready) {
> + return -EINVAL;
> + }
> +
> + spin_lock_irqsave(&kiq->ring_lock, flags);
> +
> + amdgpu_ring_alloc(ring, 32);
> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req
> + eng,
> + hub->vm_inv_eng0_ack + eng,
> + req, 1 << vmid);
> + amdgpu_fence_emit_polling(ring, &seq);
> + amdgpu_ring_commit(ring);
> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> +
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> +
> + /* don't wait anymore for gpu reset case because this way may
> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> + * never return if we keep waiting in virt_kiq_rreg, which cause
> + * gpu_recover() hang there.
> + *
> + * also don't wait anymore for IRQ context
> + * */
> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> + goto failed_kiq;
> +
> + if (in_interrupt())
> + might_sleep();
> +
> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> + r = amdgpu_fence_wait_polling(ring, seq,
> MAX_KIQ_REG_WAIT);
> + }
> +
> + if (cnt > MAX_KIQ_REG_TRY)
> + goto failed_kiq;
> +
> + return 0;
> +
> +failed_kiq:
> + pr_err("failed to invalidate tlb with kiq\n");
> + return r;
> +}
> +
> /*
> * GART
> * VMID 0 is the physical GPU addresses as used by the kernel.
> @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> amdgpu_device *adev,
> /* Use register 17 for GART */
> const unsigned eng = 17;
> unsigned i, j;
> + int r;
>
> spin_lock(&adev->gmc.invalidate_lock);
>
> @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> amdgpu_device *adev,
> struct amdgpu_vmhub *hub = &adev->vmhub[i];
> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>
> + if (amdgpu_sriov_vf(adev) ) {
> + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp,
> vmid);
> + if (!r)
> + continue;
> + }
> +
> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>
> /* Busy wait for ACK.*/
> --
> 2.7.4
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <BYAPR12MB27755A475CE994215D964F04FB3F0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
@ 2018-08-15 14:13 ` Alex Deucher
[not found] ` <CADnq5_Pu4kxrjUK8e--EjLR55hE-eeb89KosZXajN-u3ZU2ZMA-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Alex Deucher @ 2018-08-15 14:13 UTC (permalink / raw)
To: Rex Zhu; +Cc: Emily Deng, amd-gfx list
On Wed, Aug 15, 2018 at 9:56 AM Zhu, Rex <Rex.Zhu@amd.com> wrote:
>
>
>
> > -----Original Message-----
> > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Emily
> > Deng
> > Sent: Wednesday, August 15, 2018 5:48 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Deng, Emily <Emily.Deng@amd.com>
> > Subject: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
> > invalidate tlb
> >
> > To avoid the tlb flush not interrupted by world switch, use kiq and one
> > command to do tlb invalidate.
> >
> > v2:
> > Add firmware version checking.
> >
> > v3:
> > Refine the code, and move the firmware
> > checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
> >
> > SWDEV-161497
>
> The "SWDEV-161497" is meanless.
> you can describe the issue or just remove the bug number.
>
> >
> > Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> > drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
> > ++++++++++++++++++++++++++++++++
> > 4 files changed, 79 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > index 07924d4..67b584b 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
> > AMDGPU_CP_KIQ_IRQ_LAST
> > };
> >
> > +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> > +#define MAX_KIQ_REG_TRY 20
> > +
> > int amdgpu_device_ip_set_clockgating_state(void *dev,
> > enum amd_ip_block_type
> > block_type,
> > enum amd_clockgating_state state);
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > index 21adb1b6..3885636 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -22,9 +22,6 @@
> > */
> >
> > #include "amdgpu.h"
> > -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> > -#define MAX_KIQ_REG_TRY 20
> >
> > uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) { diff --git
> > a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > index 76d979e..c9b3db4 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > @@ -4348,8 +4348,21 @@ static void
> > gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> > uint32_t ref, uint32_t mask)
> > {
> > int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> > + struct amdgpu_device *adev = ring->adev;
> > + bool fw_version_ok = false;
> >
> > - if (amdgpu_sriov_vf(ring->adev))
> > + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> > + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
> > >gfx.me_feature_version >= 42))
> > + if ((adev->gfx.pfp_fw_version >= 0x000000b1) &&
> > (adev->gfx.pfp_feature_version >= 42))
> > + fw_version_ok = true;
> > + } else {
> > + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev-
> > >gfx.mec_feature_version >= 42))
> > + fw_version_ok = true;
> > + }
>
> Maybe we can add a flag and set the flag when request_firmware.
I was going to suggest the same thing.
>
> > +
> > + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok
Also is this specific to vega10 or do all gfx9 parts have this fix?
Please verify.
Alex
> > :
> > +false;
> > +
> > + if (amdgpu_sriov_vf(adev) && fw_version_ok)
> > gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> > ref, mask, 0x20);
> > else
> > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > index ed467de..3419178 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > @@ -311,6 +311,60 @@ static uint32_t
> > gmc_v9_0_get_invalidate_req(unsigned int vmid)
> > return req;
> > }
> >
> > +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct
> > amdgpu_vmhub *hub,
> > + unsigned eng, u32 req, uint32_t vmid) {
> > + signed long r, cnt = 0;
> > + unsigned long flags;
> > + uint32_t seq;
> > + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > + struct amdgpu_ring *ring = &kiq->ring;
> > +
> > + if (!ring->ready) {
> > + return -EINVAL;
> > + }
> > +
> > + spin_lock_irqsave(&kiq->ring_lock, flags);
> > +
> > + amdgpu_ring_alloc(ring, 32);
> > + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req
> > + eng,
> > + hub->vm_inv_eng0_ack + eng,
> > + req, 1 << vmid);
> > + amdgpu_fence_emit_polling(ring, &seq);
> > + amdgpu_ring_commit(ring);
> > + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> > +
> > + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> > +
> > + /* don't wait anymore for gpu reset case because this way may
> > + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> > + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> > + * never return if we keep waiting in virt_kiq_rreg, which cause
> > + * gpu_recover() hang there.
> > + *
> > + * also don't wait anymore for IRQ context
> > + * */
> > + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> > + goto failed_kiq;
> > +
> > + if (in_interrupt())
> > + might_sleep();
> > +
> > + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> > + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> > + r = amdgpu_fence_wait_polling(ring, seq,
> > MAX_KIQ_REG_WAIT);
> > + }
> > +
> > + if (cnt > MAX_KIQ_REG_TRY)
> > + goto failed_kiq;
> > +
> > + return 0;
> > +
> > +failed_kiq:
> > + pr_err("failed to invalidate tlb with kiq\n");
> > + return r;
> > +}
> > +
> > /*
> > * GART
> > * VMID 0 is the physical GPU addresses as used by the kernel.
> > @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > amdgpu_device *adev,
> > /* Use register 17 for GART */
> > const unsigned eng = 17;
> > unsigned i, j;
> > + int r;
> >
> > spin_lock(&adev->gmc.invalidate_lock);
> >
> > @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > amdgpu_device *adev,
> > struct amdgpu_vmhub *hub = &adev->vmhub[i];
> > u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
> >
> > + if (amdgpu_sriov_vf(adev) ) {
> > + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp,
> > vmid);
> > + if (!r)
> > + continue;
> > + }
> > +
> > WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
> >
> > /* Busy wait for ACK.*/
> > --
> > 2.7.4
> >
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <CADnq5_Pu4kxrjUK8e--EjLR55hE-eeb89KosZXajN-u3ZU2ZMA-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
@ 2018-08-16 5:41 ` Zhu, Rex
[not found] ` <BYAPR12MB27754BF4F6A7F1560D93DE01FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Zhu, Rex @ 2018-08-16 5:41 UTC (permalink / raw)
To: 'Alex Deucher'; +Cc: Deng, Emily, amd-gfx list
> -----Original Message-----
> From: Alex Deucher <alexdeucher@gmail.com>
> Sent: Wednesday, August 15, 2018 10:14 PM
> To: Zhu, Rex <Rex.Zhu@amd.com>
> Cc: Deng, Emily <Emily.Deng@amd.com>; amd-gfx list <amd-
> gfx@lists.freedesktop.org>
> Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
> invalidate tlb
>
> On Wed, Aug 15, 2018 at 9:56 AM Zhu, Rex <Rex.Zhu@amd.com> wrote:
> >
> >
> >
> > > -----Original Message-----
> > > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
> > > Emily Deng
> > > Sent: Wednesday, August 15, 2018 5:48 PM
> > > To: amd-gfx@lists.freedesktop.org
> > > Cc: Deng, Emily <Emily.Deng@amd.com>
> > > Subject: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to
> > > do invalidate tlb
> > >
> > > To avoid the tlb flush not interrupted by world switch, use kiq and
> > > one command to do tlb invalidate.
> > >
> > > v2:
> > > Add firmware version checking.
> > >
> > > v3:
> > > Refine the code, and move the firmware checking into
> > > gfx_v9_0_ring_emit_reg_write_reg_wait.
> > >
> > > SWDEV-161497
> >
> > The "SWDEV-161497" is meanless.
> > you can describe the issue or just remove the bug number.
> >
> > >
> > > Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> > > ---
> > > drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> > > drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> > > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> > > drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
> > > ++++++++++++++++++++++++++++++++
> > > 4 files changed, 79 insertions(+), 4 deletions(-)
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > index 07924d4..67b584b 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
> > > AMDGPU_CP_KIQ_IRQ_LAST
> > > };
> > >
> > > +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > > +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> > > +#define MAX_KIQ_REG_TRY 20
> > > +
> > > int amdgpu_device_ip_set_clockgating_state(void *dev,
> > > enum amd_ip_block_type
> > > block_type,
> > > enum amd_clockgating_state
> > > state); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > index 21adb1b6..3885636 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > @@ -22,9 +22,6 @@
> > > */
> > >
> > > #include "amdgpu.h"
> > > -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > > -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ -
> #define
> > > MAX_KIQ_REG_TRY 20
> > >
> > > uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) { diff --git
> > > a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > index 76d979e..c9b3db4 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > @@ -4348,8 +4348,21 @@ static void
> > > gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> > > uint32_t ref,
> > > uint32_t mask) {
> > > int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> > > + struct amdgpu_device *adev = ring->adev;
> > > + bool fw_version_ok = false;
> > >
> > > - if (amdgpu_sriov_vf(ring->adev))
> > > + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> > > + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev-
> > > >gfx.me_feature_version >= 42))
> > > + if ((adev->gfx.pfp_fw_version >= 0x000000b1)
> > > + &&
> > > (adev->gfx.pfp_feature_version >= 42))
> > > + fw_version_ok = true;
> > > + } else {
> > > + if ((adev->gfx.mec_fw_version >= 0x00000193) &&
> > > + (adev-
> > > >gfx.mec_feature_version >= 42))
> > > + fw_version_ok = true;
> > > + }
> >
> > Maybe we can add a flag and set the flag when request_firmware.
>
> I was going to suggest the same thing.
>
>
> >
> > > +
> > > + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ?
> > > + fw_version_ok
>
> Also is this specific to vega10 or do all gfx9 parts have this fix?
> Please verify.
>
> Alex
The patch can work on Raven.
Raven firmware version as :
MC feature version: 0, firmware version: 0x00000000
ME feature version: 42, firmware version: 0x0000009c
PFP feature version: 42, firmware version: 0x000000b1
CE feature version: 42, firmware version: 0x0000004d
MEC feature version: 42, firmware version: 0x00000192
MEC2 feature version: 42, firmware version: 0x00000192
VBIOS version: 113-RAVEN-110
Best Regards
Rex
> > > :
> > > +false;
> > > +
> > > + if (amdgpu_sriov_vf(adev) && fw_version_ok)
> > > gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> > > ref, mask, 0x20);
> > > else
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > index ed467de..3419178 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > @@ -311,6 +311,60 @@ static uint32_t
> > > gmc_v9_0_get_invalidate_req(unsigned int vmid)
> > > return req;
> > > }
> > >
> > > +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev,
> > > +struct
> > > amdgpu_vmhub *hub,
> > > + unsigned eng, u32 req, uint32_t vmid) {
> > > + signed long r, cnt = 0;
> > > + unsigned long flags;
> > > + uint32_t seq;
> > > + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > > + struct amdgpu_ring *ring = &kiq->ring;
> > > +
> > > + if (!ring->ready) {
> > > + return -EINVAL;
> > > + }
> > > +
> > > + spin_lock_irqsave(&kiq->ring_lock, flags);
> > > +
> > > + amdgpu_ring_alloc(ring, 32);
> > > + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req
> > > + eng,
> > > + hub->vm_inv_eng0_ack + eng,
> > > + req, 1 << vmid);
> > > + amdgpu_fence_emit_polling(ring, &seq);
> > > + amdgpu_ring_commit(ring);
> > > + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> > > +
> > > + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> > > +
> > > + /* don't wait anymore for gpu reset case because this way may
> > > + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> > > + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> > > + * never return if we keep waiting in virt_kiq_rreg, which cause
> > > + * gpu_recover() hang there.
> > > + *
> > > + * also don't wait anymore for IRQ context
> > > + * */
> > > + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> > > + goto failed_kiq;
> > > +
> > > + if (in_interrupt())
> > > + might_sleep();
> > > +
> > > + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> > > + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> > > + r = amdgpu_fence_wait_polling(ring, seq,
> > > MAX_KIQ_REG_WAIT);
> > > + }
> > > +
> > > + if (cnt > MAX_KIQ_REG_TRY)
> > > + goto failed_kiq;
> > > +
> > > + return 0;
> > > +
> > > +failed_kiq:
> > > + pr_err("failed to invalidate tlb with kiq\n");
> > > + return r;
> > > +}
> > > +
> > > /*
> > > * GART
> > > * VMID 0 is the physical GPU addresses as used by the kernel.
> > > @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > > amdgpu_device *adev,
> > > /* Use register 17 for GART */
> > > const unsigned eng = 17;
> > > unsigned i, j;
> > > + int r;
> > >
> > > spin_lock(&adev->gmc.invalidate_lock);
> > >
> > > @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > > amdgpu_device *adev,
> > > struct amdgpu_vmhub *hub = &adev->vmhub[i];
> > > u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
> > >
> > > + if (amdgpu_sriov_vf(adev) ) {
> > > + r = amdgpu_kiq_invalidate_tlb(adev, hub, eng,
> > > + tmp,
> > > vmid);
> > > + if (!r)
> > > + continue;
> > > + }
> > > +
> > > WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
> > >
> > > /* Busy wait for ACK.*/
> > > --
> > > 2.7.4
> > >
> > > _______________________________________________
> > > amd-gfx mailing list
> > > amd-gfx@lists.freedesktop.org
> > > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <BYAPR12MB27754BF4F6A7F1560D93DE01FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
@ 2018-08-16 6:24 ` Zhu, Rex
[not found] ` <BYAPR12MB2775F83981EF5557A7613128FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Zhu, Rex @ 2018-08-16 6:24 UTC (permalink / raw)
To: 'Alex Deucher'; +Cc: Deng, Emily, 'amd-gfx list'
> -----Original Message-----
> From: Zhu, Rex
> Sent: Thursday, August 16, 2018 1:41 PM
> To: 'Alex Deucher' <alexdeucher@gmail.com>
> Cc: Deng, Emily <Emily.Deng@amd.com>; amd-gfx list <amd-
> gfx@lists.freedesktop.org>
> Subject: RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
> invalidate tlb
>
>
>
> > -----Original Message-----
> > From: Alex Deucher <alexdeucher@gmail.com>
> > Sent: Wednesday, August 15, 2018 10:14 PM
> > To: Zhu, Rex <Rex.Zhu@amd.com>
> > Cc: Deng, Emily <Emily.Deng@amd.com>; amd-gfx list <amd-
> > gfx@lists.freedesktop.org>
> > Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq
> > to do invalidate tlb
> >
> > On Wed, Aug 15, 2018 at 9:56 AM Zhu, Rex <Rex.Zhu@amd.com> wrote:
> > >
> > >
> > >
> > > > -----Original Message-----
> > > > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of
> > > > Emily Deng
> > > > Sent: Wednesday, August 15, 2018 5:48 PM
> > > > To: amd-gfx@lists.freedesktop.org
> > > > Cc: Deng, Emily <Emily.Deng@amd.com>
> > > > Subject: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq
> > > > to do invalidate tlb
> > > >
> > > > To avoid the tlb flush not interrupted by world switch, use kiq
> > > > and one command to do tlb invalidate.
> > > >
> > > > v2:
> > > > Add firmware version checking.
> > > >
> > > > v3:
> > > > Refine the code, and move the firmware checking into
> > > > gfx_v9_0_ring_emit_reg_write_reg_wait.
> > > >
> > > > SWDEV-161497
> > >
> > > The "SWDEV-161497" is meanless.
> > > you can describe the issue or just remove the bug number.
> > >
> > > >
> > > > Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> > > > ---
> > > > drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> > > > drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> > > > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> > > > drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
> > > > ++++++++++++++++++++++++++++++++
> > > > 4 files changed, 79 insertions(+), 4 deletions(-)
> > > >
> > > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > > b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > > index 07924d4..67b584b 100644
> > > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> > > > @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
> > > > AMDGPU_CP_KIQ_IRQ_LAST
> > > > };
> > > >
> > > > +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > > > +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> > > > +#define MAX_KIQ_REG_TRY 20
> > > > +
> > > > int amdgpu_device_ip_set_clockgating_state(void *dev,
> > > > enum amd_ip_block_type
> > > > block_type,
> > > > enum
> > > > amd_clockgating_state state); diff --git
> > > > a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > > index 21adb1b6..3885636 100644
> > > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > > > @@ -22,9 +22,6 @@
> > > > */
> > > >
> > > > #include "amdgpu.h"
> > > > -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> > > > -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ -
> > #define
> > > > MAX_KIQ_REG_TRY 20
> > > >
> > > > uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) { diff
> > > > --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > > index 76d979e..c9b3db4 100644
> > > > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> > > > @@ -4348,8 +4348,21 @@ static void
> > > > gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> > > > uint32_t ref,
> > > > uint32_t mask) {
> > > > int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> > > > + struct amdgpu_device *adev = ring->adev;
> > > > + bool fw_version_ok = false;
> > > >
> > > > - if (amdgpu_sriov_vf(ring->adev))
> > > > + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> > > > + if ((adev->gfx.me_fw_version >= 0x0000009c) &&
> > > > + (adev-
> > > > >gfx.me_feature_version >= 42))
> > > > + if ((adev->gfx.pfp_fw_version >=
> > > > + 0x000000b1) &&
> > > > (adev->gfx.pfp_feature_version >= 42))
> > > > + fw_version_ok = true;
> > > > + } else {
> > > > + if ((adev->gfx.mec_fw_version >= 0x00000193) &&
> > > > + (adev-
> > > > >gfx.mec_feature_version >= 42))
> > > > + fw_version_ok = true;
> > > > + }
> > >
> > > Maybe we can add a flag and set the flag when request_firmware.
> >
> > I was going to suggest the same thing.
> >
> >
> > >
> > > > +
> > > > + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ?
> > > > + fw_version_ok
> >
> > Also is this specific to vega10 or do all gfx9 parts have this fix?
> > Please verify.
> >
> > Alex
>
> The patch can work on Raven.
> Raven firmware version as :
> MC feature version: 0, firmware version: 0x00000000 ME feature version: 42,
> firmware version: 0x0000009c PFP feature version: 42, firmware version:
> 0x000000b1 CE feature version: 42, firmware version: 0x0000004d MEC
> feature version: 42, firmware version: 0x00000192
> MEC2 feature version: 42, firmware version: 0x00000192 VBIOS version: 113-
> RAVEN-110
The patch can work on Vega12.
Firmware version on Vega12:
MC feature version: 0, firmware version: 0x00000000
ME feature version: 44, firmware version: 0x0000009c
PFP feature version: 44, firmware version: 0x000000b2
CE feature version: 44, firmware version: 0x0000004d
MEC feature version: 44, firmware version: 0x00000196
MEC2 feature version: 44, firmware version: 0x00000196
VBIOS version: 113-D2060300-016
Best Regards
Rex
>
>
> > > > :
> > > > +false;
> > > > +
> > > > + if (amdgpu_sriov_vf(adev) && fw_version_ok)
> > > > gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> > > > ref, mask, 0x20);
> > > > else
> > > > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > > b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > > index ed467de..3419178 100644
> > > > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> > > > @@ -311,6 +311,60 @@ static uint32_t
> > > > gmc_v9_0_get_invalidate_req(unsigned int vmid)
> > > > return req;
> > > > }
> > > >
> > > > +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device
> > > > +*adev, struct
> > > > amdgpu_vmhub *hub,
> > > > + unsigned eng, u32 req, uint32_t vmid) {
> > > > + signed long r, cnt = 0;
> > > > + unsigned long flags;
> > > > + uint32_t seq;
> > > > + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > > > + struct amdgpu_ring *ring = &kiq->ring;
> > > > +
> > > > + if (!ring->ready) {
> > > > + return -EINVAL;
> > > > + }
> > > > +
> > > > + spin_lock_irqsave(&kiq->ring_lock, flags);
> > > > +
> > > > + amdgpu_ring_alloc(ring, 32);
> > > > + amdgpu_ring_emit_reg_write_reg_wait(ring,
> > > > + hub->vm_inv_eng0_req eng,
> > > > + hub->vm_inv_eng0_ack + eng,
> > > > + req, 1 << vmid);
> > > > + amdgpu_fence_emit_polling(ring, &seq);
> > > > + amdgpu_ring_commit(ring);
> > > > + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> > > > +
> > > > + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> > > > +
> > > > + /* don't wait anymore for gpu reset case because this way may
> > > > + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> > > > + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> > > > + * never return if we keep waiting in virt_kiq_rreg, which cause
> > > > + * gpu_recover() hang there.
> > > > + *
> > > > + * also don't wait anymore for IRQ context
> > > > + * */
> > > > + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> > > > + goto failed_kiq;
> > > > +
> > > > + if (in_interrupt())
> > > > + might_sleep();
> > > > +
> > > > + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> > > > + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> > > > + r = amdgpu_fence_wait_polling(ring, seq,
> > > > MAX_KIQ_REG_WAIT);
> > > > + }
> > > > +
> > > > + if (cnt > MAX_KIQ_REG_TRY)
> > > > + goto failed_kiq;
> > > > +
> > > > + return 0;
> > > > +
> > > > +failed_kiq:
> > > > + pr_err("failed to invalidate tlb with kiq\n");
> > > > + return r;
> > > > +}
> > > > +
> > > > /*
> > > > * GART
> > > > * VMID 0 is the physical GPU addresses as used by the kernel.
> > > > @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > > > amdgpu_device *adev,
> > > > /* Use register 17 for GART */
> > > > const unsigned eng = 17;
> > > > unsigned i, j;
> > > > + int r;
> > > >
> > > > spin_lock(&adev->gmc.invalidate_lock);
> > > >
> > > > @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
> > > > amdgpu_device *adev,
> > > > struct amdgpu_vmhub *hub = &adev->vmhub[i];
> > > > u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
> > > >
> > > > + if (amdgpu_sriov_vf(adev) ) {
> > > > + r = amdgpu_kiq_invalidate_tlb(adev, hub,
> > > > + eng, tmp,
> > > > vmid);
> > > > + if (!r)
> > > > + continue;
> > > > + }
> > > > +
> > > > WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
> > > >
> > > > /* Busy wait for ACK.*/
> > > > --
> > > > 2.7.4
> > > >
> > > > _______________________________________________
> > > > amd-gfx mailing list
> > > > amd-gfx@lists.freedesktop.org
> > > > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> > > _______________________________________________
> > > amd-gfx mailing list
> > > amd-gfx@lists.freedesktop.org
> > > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <BYAPR12MB2775F83981EF5557A7613128FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
@ 2018-08-16 7:33 ` Deng, Emily
0 siblings, 0 replies; 15+ messages in thread
From: Deng, Emily @ 2018-08-16 7:33 UTC (permalink / raw)
To: Zhu, Rex, 'Alex Deucher'; +Cc: 'amd-gfx list'
Thanks all, I am modifying the patch, and testing.
Best wishes
Emily Deng
>-----Original Message-----
>From: Zhu, Rex
>Sent: Thursday, August 16, 2018 2:25 PM
>To: 'Alex Deucher' <alexdeucher@gmail.com>
>Cc: Deng, Emily <Emily.Deng@amd.com>; 'amd-gfx list' <amd-
>gfx@lists.freedesktop.org>
>Subject: RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
>invalidate tlb
>
>
>
>> -----Original Message-----
>> From: Zhu, Rex
>> Sent: Thursday, August 16, 2018 1:41 PM
>> To: 'Alex Deucher' <alexdeucher@gmail.com>
>> Cc: Deng, Emily <Emily.Deng@amd.com>; amd-gfx list <amd-
>> gfx@lists.freedesktop.org>
>> Subject: RE: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq
>> to do invalidate tlb
>>
>>
>>
>> > -----Original Message-----
>> > From: Alex Deucher <alexdeucher@gmail.com>
>> > Sent: Wednesday, August 15, 2018 10:14 PM
>> > To: Zhu, Rex <Rex.Zhu@amd.com>
>> > Cc: Deng, Emily <Emily.Deng@amd.com>; amd-gfx list <amd-
>> > gfx@lists.freedesktop.org>
>> > Subject: Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use
>> > kiq to do invalidate tlb
>> >
>> > On Wed, Aug 15, 2018 at 9:56 AM Zhu, Rex <Rex.Zhu@amd.com> wrote:
>> > >
>> > >
>> > >
>> > > > -----Original Message-----
>> > > > From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf
>> > > > Of Emily Deng
>> > > > Sent: Wednesday, August 15, 2018 5:48 PM
>> > > > To: amd-gfx@lists.freedesktop.org
>> > > > Cc: Deng, Emily <Emily.Deng@amd.com>
>> > > > Subject: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use
>> > > > kiq to do invalidate tlb
>> > > >
>> > > > To avoid the tlb flush not interrupted by world switch, use kiq
>> > > > and one command to do tlb invalidate.
>> > > >
>> > > > v2:
>> > > > Add firmware version checking.
>> > > >
>> > > > v3:
>> > > > Refine the code, and move the firmware checking into
>> > > > gfx_v9_0_ring_emit_reg_write_reg_wait.
>> > > >
>> > > > SWDEV-161497
>> > >
>> > > The "SWDEV-161497" is meanless.
>> > > you can describe the issue or just remove the bug number.
>> > >
>> > > >
>> > > > Signed-off-by: Emily Deng <Emily.Deng@amd.com>
>> > > > ---
>> > > > drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
>> > > > drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
>> > > > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
>> > > > drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61
>> > > > ++++++++++++++++++++++++++++++++
>> > > > 4 files changed, 79 insertions(+), 4 deletions(-)
>> > > >
>> > > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> > > > b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> > > > index 07924d4..67b584b 100644
>> > > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> > > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> > > > @@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
>> > > > AMDGPU_CP_KIQ_IRQ_LAST
>> > > > };
>> > > >
>> > > > +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>> > > > +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
>> > > > +#define MAX_KIQ_REG_TRY 20
>> > > > +
>> > > > int amdgpu_device_ip_set_clockgating_state(void *dev,
>> > > > enum amd_ip_block_type
>> > > > block_type,
>> > > > enum
>> > > > amd_clockgating_state state); diff --git
>> > > > a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> > > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> > > > index 21adb1b6..3885636 100644
>> > > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> > > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> > > > @@ -22,9 +22,6 @@
>> > > > */
>> > > >
>> > > > #include "amdgpu.h"
>> > > > -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
>> > > > -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ -
>> > #define
>> > > > MAX_KIQ_REG_TRY 20
>> > > >
>> > > > uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) { diff
>> > > > --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> > > > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> > > > index 76d979e..c9b3db4 100644
>> > > > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> > > > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> > > > @@ -4348,8 +4348,21 @@ static void
>> > > > gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
>> > > > uint32_t ref,
>> > > > uint32_t mask) {
>> > > > int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
>> > > > + struct amdgpu_device *adev = ring->adev;
>> > > > + bool fw_version_ok = false;
>> > > >
>> > > > - if (amdgpu_sriov_vf(ring->adev))
>> > > > + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
>> > > > + if ((adev->gfx.me_fw_version >= 0x0000009c) &&
>> > > > + (adev-
>> > > > >gfx.me_feature_version >= 42))
>> > > > + if ((adev->gfx.pfp_fw_version >=
>> > > > + 0x000000b1) &&
>> > > > (adev->gfx.pfp_feature_version >= 42))
>> > > > + fw_version_ok = true;
>> > > > + } else {
>> > > > + if ((adev->gfx.mec_fw_version >= 0x00000193) &&
>> > > > + (adev-
>> > > > >gfx.mec_feature_version >= 42))
>> > > > + fw_version_ok = true;
>> > > > + }
>> > >
>> > > Maybe we can add a flag and set the flag when request_firmware.
>> >
>> > I was going to suggest the same thing.
>> >
>> >
>> > >
>> > > > +
>> > > > + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ?
>> > > > + fw_version_ok
>> >
>> > Also is this specific to vega10 or do all gfx9 parts have this fix?
>> > Please verify.
>> >
>> > Alex
>>
>> The patch can work on Raven.
>> Raven firmware version as :
>> MC feature version: 0, firmware version: 0x00000000 ME feature
>> version: 42, firmware version: 0x0000009c PFP feature version: 42, firmware
>version:
>> 0x000000b1 CE feature version: 42, firmware version: 0x0000004d MEC
>> feature version: 42, firmware version: 0x00000192
>> MEC2 feature version: 42, firmware version: 0x00000192 VBIOS version:
>> 113-
>> RAVEN-110
>
>The patch can work on Vega12.
>Firmware version on Vega12:
>MC feature version: 0, firmware version: 0x00000000 ME feature version: 44,
>firmware version: 0x0000009c PFP feature version: 44, firmware version:
>0x000000b2 CE feature version: 44, firmware version: 0x0000004d MEC feature
>version: 44, firmware version: 0x00000196
>MEC2 feature version: 44, firmware version: 0x00000196 VBIOS version: 113-
>D2060300-016
>
>
>Best Regards
>Rex
>>
>>
>> > > > :
>> > > > +false;
>> > > > +
>> > > > + if (amdgpu_sriov_vf(adev) && fw_version_ok)
>> > > > gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
>> > > > ref, mask, 0x20);
>> > > > else
>> > > > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> > > > b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> > > > index ed467de..3419178 100644
>> > > > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> > > > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
>> > > > @@ -311,6 +311,60 @@ static uint32_t
>> > > > gmc_v9_0_get_invalidate_req(unsigned int vmid)
>> > > > return req;
>> > > > }
>> > > >
>> > > > +signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device
>> > > > +*adev, struct
>> > > > amdgpu_vmhub *hub,
>> > > > + unsigned eng, u32 req, uint32_t vmid) {
>> > > > + signed long r, cnt = 0;
>> > > > + unsigned long flags;
>> > > > + uint32_t seq;
>> > > > + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> > > > + struct amdgpu_ring *ring = &kiq->ring;
>> > > > +
>> > > > + if (!ring->ready) {
>> > > > + return -EINVAL;
>> > > > + }
>> > > > +
>> > > > + spin_lock_irqsave(&kiq->ring_lock, flags);
>> > > > +
>> > > > + amdgpu_ring_alloc(ring, 32);
>> > > > + amdgpu_ring_emit_reg_write_reg_wait(ring,
>> > > > + hub->vm_inv_eng0_req eng,
>> > > > + hub->vm_inv_eng0_ack + eng,
>> > > > + req, 1 << vmid);
>> > > > + amdgpu_fence_emit_polling(ring, &seq);
>> > > > + amdgpu_ring_commit(ring);
>> > > > + spin_unlock_irqrestore(&kiq->ring_lock, flags);
>> > > > +
>> > > > + r = amdgpu_fence_wait_polling(ring, seq,
>> > > > + MAX_KIQ_REG_WAIT);
>> > > > +
>> > > > + /* don't wait anymore for gpu reset case because this way may
>> > > > + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
>> > > > + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
>> > > > + * never return if we keep waiting in virt_kiq_rreg, which cause
>> > > > + * gpu_recover() hang there.
>> > > > + *
>> > > > + * also don't wait anymore for IRQ context
>> > > > + * */
>> > > > + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
>> > > > + goto failed_kiq;
>> > > > +
>> > > > + if (in_interrupt())
>> > > > + might_sleep();
>> > > > +
>> > > > + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
>> > > > + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
>> > > > + r = amdgpu_fence_wait_polling(ring, seq,
>> > > > MAX_KIQ_REG_WAIT);
>> > > > + }
>> > > > +
>> > > > + if (cnt > MAX_KIQ_REG_TRY)
>> > > > + goto failed_kiq;
>> > > > +
>> > > > + return 0;
>> > > > +
>> > > > +failed_kiq:
>> > > > + pr_err("failed to invalidate tlb with kiq\n");
>> > > > + return r;
>> > > > +}
>> > > > +
>> > > > /*
>> > > > * GART
>> > > > * VMID 0 is the physical GPU addresses as used by the kernel.
>> > > > @@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>> > > > amdgpu_device *adev,
>> > > > /* Use register 17 for GART */
>> > > > const unsigned eng = 17;
>> > > > unsigned i, j;
>> > > > + int r;
>> > > >
>> > > > spin_lock(&adev->gmc.invalidate_lock);
>> > > >
>> > > > @@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct
>> > > > amdgpu_device *adev,
>> > > > struct amdgpu_vmhub *hub = &adev->vmhub[i];
>> > > > u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>> > > >
>> > > > + if (amdgpu_sriov_vf(adev) ) {
>> > > > + r = amdgpu_kiq_invalidate_tlb(adev, hub,
>> > > > + eng, tmp,
>> > > > vmid);
>> > > > + if (!r)
>> > > > + continue;
>> > > > + }
>> > > > +
>> > > > WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>> > > >
>> > > > /* Busy wait for ACK.*/
>> > > > --
>> > > > 2.7.4
>> > > >
>> > > > _______________________________________________
>> > > > amd-gfx mailing list
>> > > > amd-gfx@lists.freedesktop.org
>> > > > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>> > > _______________________________________________
>> > > amd-gfx mailing list
>> > > amd-gfx@lists.freedesktop.org
>> > > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
[not found] ` <1534337675-25039-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
@ 2018-08-15 13:09 ` Christian König
0 siblings, 0 replies; 15+ messages in thread
From: Christian König @ 2018-08-15 13:09 UTC (permalink / raw)
To: Emily Deng, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW
Am 15.08.2018 um 14:54 schrieb Emily Deng:
> To avoid the tlb flush not interrupted by world switch, use kiq and one
> command to do tlb invalidate.
>
> v2:
> Add firmware version checking.
>
> v3:
> Refine the code, and move the firmware
> checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
>
> v4:
> Change the name amdgpu_kiq_invalidate_tlb to amdgpu_kiq_reg_write_reg_wait.
> Remove the in_interrupt.
> Refine code.
>
> SWDEV-161497
>
> Signed-off-by: Emily Deng <Emily.Deng@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 60 ++++++++++++++++++++++++++++++++
> 4 files changed, 78 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 6265b88..19ef7711 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -212,6 +212,10 @@ enum amdgpu_kiq_irq {
> AMDGPU_CP_KIQ_IRQ_LAST
> };
>
> +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> +#define MAX_KIQ_REG_TRY 20
> +
> int amdgpu_device_ip_set_clockgating_state(void *dev,
> enum amd_ip_block_type block_type,
> enum amd_clockgating_state state);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 21adb1b6..3885636 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -22,9 +22,6 @@
> */
>
> #include "amdgpu.h"
> -#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
> -#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
> -#define MAX_KIQ_REG_TRY 20
>
> uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
> {
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 76d979e..e010166 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -4348,8 +4348,21 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
> uint32_t ref, uint32_t mask)
> {
> int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
> + struct amdgpu_device *adev = ring->adev;
> + bool fw_version_ok = false;
>
> - if (amdgpu_sriov_vf(ring->adev))
> + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
> + if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev->gfx.me_feature_version >= 42)
> + && (adev->gfx.pfp_fw_version >= 0x000000b1) && (adev->gfx.pfp_feature_version >= 42))
> + fw_version_ok = true;
The indentation here is wrong and the && belongs at the end of the line.
> + } else {
> + if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev->gfx.mec_feature_version >= 42))
> + fw_version_ok = true;
> + }
> +
> + fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok : false;
> +
> + if (amdgpu_sriov_vf(adev) && fw_version_ok)
Please squash the second patch into this one. The goal is to use the
same code path on both SRIOV and bare metal.
I would rather separate our this change into a patch and then make the
TLB invalidation the second patch.
> gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
> ref, mask, 0x20);
> else
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index ed467de..9726c7e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -311,6 +311,59 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
> return req;
> }
>
> +signed long amdgpu_kiq_reg_write_reg_wait(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
> + unsigned eng, u32 req, uint32_t vmid)
That is still specialized, e.g. you assume that you work with a VMHUB
here instead of specifying the registers, req and value separately.
> +{
> + signed long r, cnt = 0;
> + unsigned long flags;
> + uint32_t seq;
> + struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> + struct amdgpu_ring *ring = &kiq->ring;
> +
> + if (!ring->ready) {
> + return -EINVAL;
> + }
Please drop the {} here.
Christian.
> +
> + spin_lock_irqsave(&kiq->ring_lock, flags);
> +
> + amdgpu_ring_alloc(ring, 32);
> + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
> + hub->vm_inv_eng0_ack + eng,
> + req, 1 << vmid);
> + amdgpu_fence_emit_polling(ring, &seq);
> + amdgpu_ring_commit(ring);
> + spin_unlock_irqrestore(&kiq->ring_lock, flags);
> +
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> +
> + /* don't wait anymore for gpu reset case because this way may
> + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
> + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
> + * never return if we keep waiting in virt_kiq_rreg, which cause
> + * gpu_recover() hang there.
> + *
> + * also don't wait anymore for IRQ context
> + * */
> + if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
> + goto failed_kiq;
> +
> + might_sleep();
> +
> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> + }
> +
> + if (cnt > MAX_KIQ_REG_TRY)
> + goto failed_kiq;
> +
> + return 0;
> +
> +failed_kiq:
> + pr_err("failed to invalidate tlb with kiq\n");
> + return r;
> +}
> +
> /*
> * GART
> * VMID 0 is the physical GPU addresses as used by the kernel.
> @@ -332,6 +385,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
> /* Use register 17 for GART */
> const unsigned eng = 17;
> unsigned i, j;
> + int r;
>
> spin_lock(&adev->gmc.invalidate_lock);
>
> @@ -339,6 +393,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
> struct amdgpu_vmhub *hub = &adev->vmhub[i];
> u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
>
> + if (amdgpu_sriov_vf(adev)) {
> + r = amdgpu_kiq_reg_write_reg_wait(adev, hub, eng, tmp, vmid);
> + if (!r)
> + continue;
> + }
> +
> WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
>
> /* Busy wait for ACK.*/
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
@ 2018-08-15 12:54 Emily Deng
[not found] ` <1534337675-25039-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
0 siblings, 1 reply; 15+ messages in thread
From: Emily Deng @ 2018-08-15 12:54 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: Emily Deng
To avoid the tlb flush not interrupted by world switch, use kiq and one
command to do tlb invalidate.
v2:
Add firmware version checking.
v3:
Refine the code, and move the firmware
checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
v4:
Change the name amdgpu_kiq_invalidate_tlb to amdgpu_kiq_reg_write_reg_wait.
Remove the in_interrupt.
Refine code.
SWDEV-161497
Signed-off-by: Emily Deng <Emily.Deng@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 60 ++++++++++++++++++++++++++++++++
4 files changed, 78 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 6265b88..19ef7711 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -212,6 +212,10 @@ enum amdgpu_kiq_irq {
AMDGPU_CP_KIQ_IRQ_LAST
};
+#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
+#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
+#define MAX_KIQ_REG_TRY 20
+
int amdgpu_device_ip_set_clockgating_state(void *dev,
enum amd_ip_block_type block_type,
enum amd_clockgating_state state);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 21adb1b6..3885636 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -22,9 +22,6 @@
*/
#include "amdgpu.h"
-#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
-#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
-#define MAX_KIQ_REG_TRY 20
uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 76d979e..e010166 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4348,8 +4348,21 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
uint32_t ref, uint32_t mask)
{
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+ struct amdgpu_device *adev = ring->adev;
+ bool fw_version_ok = false;
- if (amdgpu_sriov_vf(ring->adev))
+ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+ if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev->gfx.me_feature_version >= 42)
+ && (adev->gfx.pfp_fw_version >= 0x000000b1) && (adev->gfx.pfp_feature_version >= 42))
+ fw_version_ok = true;
+ } else {
+ if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev->gfx.mec_feature_version >= 42))
+ fw_version_ok = true;
+ }
+
+ fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok : false;
+
+ if (amdgpu_sriov_vf(adev) && fw_version_ok)
gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
ref, mask, 0x20);
else
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..9726c7e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -311,6 +311,59 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
return req;
}
+signed long amdgpu_kiq_reg_write_reg_wait(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+ unsigned eng, u32 req, uint32_t vmid)
+{
+ signed long r, cnt = 0;
+ unsigned long flags;
+ uint32_t seq;
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *ring = &kiq->ring;
+
+ if (!ring->ready) {
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&kiq->ring_lock, flags);
+
+ amdgpu_ring_alloc(ring, 32);
+ amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+ hub->vm_inv_eng0_ack + eng,
+ req, 1 << vmid);
+ amdgpu_fence_emit_polling(ring, &seq);
+ amdgpu_ring_commit(ring);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+ /* don't wait anymore for gpu reset case because this way may
+ * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+ * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+ * never return if we keep waiting in virt_kiq_rreg, which cause
+ * gpu_recover() hang there.
+ *
+ * also don't wait anymore for IRQ context
+ * */
+ if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+ goto failed_kiq;
+
+ might_sleep();
+
+ while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+ msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+ }
+
+ if (cnt > MAX_KIQ_REG_TRY)
+ goto failed_kiq;
+
+ return 0;
+
+failed_kiq:
+ pr_err("failed to invalidate tlb with kiq\n");
+ return r;
+}
+
/*
* GART
* VMID 0 is the physical GPU addresses as used by the kernel.
@@ -332,6 +385,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
/* Use register 17 for GART */
const unsigned eng = 17;
unsigned i, j;
+ int r;
spin_lock(&adev->gmc.invalidate_lock);
@@ -339,6 +393,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
struct amdgpu_vmhub *hub = &adev->vmhub[i];
u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
+ if (amdgpu_sriov_vf(adev)) {
+ r = amdgpu_kiq_reg_write_reg_wait(adev, hub, eng, tmp, vmid);
+ if (!r)
+ continue;
+ }
+
WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
/* Busy wait for ACK.*/
--
2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 15+ messages in thread
* [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb
@ 2018-08-15 9:52 Emily Deng
0 siblings, 0 replies; 15+ messages in thread
From: Emily Deng @ 2018-08-15 9:52 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: Emily Deng
To avoid the tlb flush not interrupted by world switch, use kiq and one
command to do tlb invalidate.
v2:
Add firmware version checking.
v3:
Refine the code, and move the firmware
checking into gfx_v9_0_ring_emit_reg_write_reg_wait.
SWDEV-161497
Signed-off-by: Emily Deng <Emily.Deng@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 --
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 +++++++-
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 61 ++++++++++++++++++++++++++++++++
4 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 07924d4..67b584b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -210,6 +210,10 @@ enum amdgpu_kiq_irq {
AMDGPU_CP_KIQ_IRQ_LAST
};
+#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
+#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
+#define MAX_KIQ_REG_TRY 20
+
int amdgpu_device_ip_set_clockgating_state(void *dev,
enum amd_ip_block_type block_type,
enum amd_clockgating_state state);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 21adb1b6..3885636 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -22,9 +22,6 @@
*/
#include "amdgpu.h"
-#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
-#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
-#define MAX_KIQ_REG_TRY 20
uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 76d979e..c9b3db4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4348,8 +4348,21 @@ static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
uint32_t ref, uint32_t mask)
{
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+ struct amdgpu_device *adev = ring->adev;
+ bool fw_version_ok = false;
- if (amdgpu_sriov_vf(ring->adev))
+ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+ if ((adev->gfx.me_fw_version >= 0x0000009c) && (adev->gfx.me_feature_version >= 42))
+ if ((adev->gfx.pfp_fw_version >= 0x000000b1) && (adev->gfx.pfp_feature_version >= 42))
+ fw_version_ok = true;
+ } else {
+ if ((adev->gfx.mec_fw_version >= 0x00000193) && (adev->gfx.mec_feature_version >= 42))
+ fw_version_ok = true;
+ }
+
+ fw_version_ok = (adev->asic_type == CHIP_VEGA10) ? fw_version_ok : false;
+
+ if (amdgpu_sriov_vf(adev) && fw_version_ok)
gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
ref, mask, 0x20);
else
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..3419178 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -311,6 +311,60 @@ static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
return req;
}
+signed long amdgpu_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+ unsigned eng, u32 req, uint32_t vmid)
+{
+ signed long r, cnt = 0;
+ unsigned long flags;
+ uint32_t seq;
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *ring = &kiq->ring;
+
+ if (!ring->ready) {
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&kiq->ring_lock, flags);
+
+ amdgpu_ring_alloc(ring, 32);
+ amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+ hub->vm_inv_eng0_ack + eng,
+ req, 1 << vmid);
+ amdgpu_fence_emit_polling(ring, &seq);
+ amdgpu_ring_commit(ring);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+ /* don't wait anymore for gpu reset case because this way may
+ * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+ * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+ * never return if we keep waiting in virt_kiq_rreg, which cause
+ * gpu_recover() hang there.
+ *
+ * also don't wait anymore for IRQ context
+ * */
+ if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+ goto failed_kiq;
+
+ if (in_interrupt())
+ might_sleep();
+
+ while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+ msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+ }
+
+ if (cnt > MAX_KIQ_REG_TRY)
+ goto failed_kiq;
+
+ return 0;
+
+failed_kiq:
+ pr_err("failed to invalidate tlb with kiq\n");
+ return r;
+}
+
/*
* GART
* VMID 0 is the physical GPU addresses as used by the kernel.
@@ -332,6 +386,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
/* Use register 17 for GART */
const unsigned eng = 17;
unsigned i, j;
+ int r;
spin_lock(&adev->gmc.invalidate_lock);
@@ -339,6 +394,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
struct amdgpu_vmhub *hub = &adev->vmhub[i];
u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
+ if (amdgpu_sriov_vf(adev) ) {
+ r = amdgpu_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+ if (!r)
+ continue;
+ }
+
WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
/* Busy wait for ACK.*/
--
2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 15+ messages in thread
end of thread, other threads:[~2018-08-16 7:33 UTC | newest]
Thread overview: 15+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-08-15 9:48 [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb Emily Deng
[not found] ` <1534326508-17477-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 9:48 ` [PATCH] drm/amdgpu: MC_VM_FB_OFFSET is pf only register Emily Deng
[not found] ` <1534326508-17477-2-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 9:52 ` Deng, Emily
2018-08-15 10:50 ` [PATCH 1/2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb Christian König
[not found] ` <5f51ffa1-1e84-dacf-840c-76bd9cbb282b-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
2018-08-15 11:08 ` Deng, Emily
[not found] ` <CY4PR12MB112541E31A354E9E94BE2FCF8F3F0-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
2018-08-15 11:14 ` Christian König
[not found] ` <8a5bff61-c03f-856a-3f0e-3497b2bd0ba8-5C7GfCeVMHo@public.gmane.org>
2018-08-15 11:24 ` Deng, Emily
2018-08-15 13:56 ` Zhu, Rex
[not found] ` <BYAPR12MB27755A475CE994215D964F04FB3F0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
2018-08-15 14:13 ` Alex Deucher
[not found] ` <CADnq5_Pu4kxrjUK8e--EjLR55hE-eeb89KosZXajN-u3ZU2ZMA-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2018-08-16 5:41 ` Zhu, Rex
[not found] ` <BYAPR12MB27754BF4F6A7F1560D93DE01FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
2018-08-16 6:24 ` Zhu, Rex
[not found] ` <BYAPR12MB2775F83981EF5557A7613128FB3E0-ZGDeBxoHBPmJeBUhB162ZQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org>
2018-08-16 7:33 ` Deng, Emily
2018-08-15 9:52 Emily Deng
2018-08-15 12:54 Emily Deng
[not found] ` <1534337675-25039-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org>
2018-08-15 13:09 ` Christian König
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.