[PATCH 1/4] drm/amdgpu: indirect register access for nv12 sriov

[PATCH 1/4] drm/amdgpu: indirect register access for nv12 sriov

[Peng Ju Zhou]

unify host driver and guest driver indirect access
control bits names

Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 097af4d3b6b1..3580a746ad9a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -110,11 +110,11 @@ enum AMDGIM_FEATURE_FLAG {
 
 enum AMDGIM_REG_ACCESS_FLAG {
 	/* Use PSP to program IH_RB_CNTL */
-	AMDGIM_FEATURE_IH_REG_PSP_EN = (1 << 0),
+	AMDGIM_FEATURE_IH_REG_PSP_EN     = (1 << 0),
 	/* Use RLC to program MMHUB regs */
-	AMDGIM_FEATURE_RLC_MMHUB_EN  = (1 << 1),
+	AMDGIM_FEATURE_MMHUB_REG_RLC_EN  = (1 << 1),
 	/* Use RLC to program GC regs */
-	AMDGIM_FEATURE_RLC_GC_EN     = (1 << 2),
+	AMDGIM_FEATURE_GC_REG_RLC_EN     = (1 << 2),
 };
 
 struct amdgim_pf2vf_info_v1 {
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 2/4] drm/amdgpu: indirect register access for nv12 sriov

[Peng Ju Zhou]

get pf2vf msg info at it's earliest time so that
guest driver can use these info to decide whether
register indirect access enabled.

Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 5 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index a501d1a4d000..060d0ae99453 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2080,6 +2080,11 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
 				return r;
 			}
+
+			/*get pf2vf msg info at it's earliest time*/
+			if (amdgpu_sriov_vf(adev))
+				amdgpu_virt_init_data_exchange(adev);
+
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index d9ffff8eb41d..b62f134d6dd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -617,6 +617,14 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 				if (adev->virt.ras_init_done)
 					amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size);
 			}
+	} else if (adev->bios != NULL) {
+		adev->virt.fw_reserve.p_pf2vf =
+			(struct amd_sriov_msg_pf2vf_info_header *)
+			(adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
+
+		amdgpu_virt_read_pf2vf_data(adev);
+
+		return;
 	}
 
 	if (adev->virt.vf2pf_update_interval_ms != 0) {
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 3/4] drm/amdgpu: indirect register access for nv12 sriov

[Peng Ju Zhou]

using the control bits got from host to control registers access.

Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c |  2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 17 +++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index b62f134d6dd5..0c9c5255aa42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -466,6 +466,8 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms;
 		adev->virt.gim_feature =
 			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all;
+		adev->virt.reg_access =
+			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all;
 
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 3580a746ad9a..383d4bdc3fb5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -228,6 +228,7 @@ struct amdgpu_virt {
 	bool tdr_debug;
 	struct amdgpu_virt_ras_err_handler_data *virt_eh_data;
 	bool ras_init_done;
+	uint32_t reg_access;
 
 	/* vf2pf message */
 	struct delayed_work vf2pf_work;
@@ -249,6 +250,22 @@ struct amdgpu_virt {
 #define amdgpu_sriov_fullaccess(adev) \
 (amdgpu_sriov_vf((adev)) && !amdgpu_sriov_runtime((adev)))
 
+#define amdgpu_sriov_reg_indirect_en(adev) \
+(amdgpu_sriov_vf((adev)) && \
+	((adev)->virt.gim_feature & (AMDGIM_FEATURE_INDIRECT_REG_ACCESS)))
+
+#define amdgpu_sriov_reg_indirect_ih(adev) \
+(amdgpu_sriov_vf((adev)) && \
+	((adev)->virt.reg_access & (AMDGIM_FEATURE_IH_REG_PSP_EN)))
+
+#define amdgpu_sriov_reg_indirect_mmhub(adev) \
+(amdgpu_sriov_vf((adev)) && \
+	((adev)->virt.reg_access & (AMDGIM_FEATURE_MMHUB_REG_RLC_EN)))
+
+#define amdgpu_sriov_reg_indirect_gc(adev) \
+(amdgpu_sriov_vf((adev)) && \
+	((adev)->virt.reg_access & (AMDGIM_FEATURE_GC_REG_RLC_EN)))
+
 #define amdgpu_passthrough(adev) \
 ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
 
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov

[Peng Ju Zhou]

1. expand rlcg interface for gc & mmhub indirect access
2. add rlcg interface for no kiq

Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h    |   3 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 131 ++++++++++++++++++---
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      |   2 +-
 drivers/gpu/drm/amd/amdgpu/soc15_common.h  |  75 ++++++------
 5 files changed, 150 insertions(+), 63 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 060d0ae99453..438e2f732377 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -490,7 +490,7 @@ void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 	    adev->gfx.rlc.funcs &&
 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
-			return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
+			return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0);
 	} else {
 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index aeaaae713c59..4fc2ce8ce8ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -127,7 +127,8 @@ struct amdgpu_rlc_funcs {
 	void (*reset)(struct amdgpu_device *adev);
 	void (*start)(struct amdgpu_device *adev);
 	void (*update_spm_vmid)(struct amdgpu_device *adev, unsigned vmid);
-	void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v);
+	void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag);
+	u32 (*rlcg_rreg)(struct amdgpu_device *adev, u32 offset, u32 flag);
 	bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg);
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index b4fd0394cd08..85a6a10e048f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -177,6 +177,11 @@
 #define mmGC_THROTTLE_CTRL_Sienna_Cichlid              0x2030
 #define mmGC_THROTTLE_CTRL_Sienna_Cichlid_BASE_IDX     0
 
+#define GFX_RLCG_GC_WRITE_OLD	(0x8 << 28)
+#define GFX_RLCG_GC_WRITE	(0x0 << 28)
+#define GFX_RLCG_GC_READ	(0x1 << 28)
+#define GFX_RLCG_MMHUB_WRITE	(0x2 << 28)
+
 MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
 MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
 MODULE_FIRMWARE("amdgpu/navi10_me.bin");
@@ -1422,38 +1427,127 @@ static const struct soc15_reg_golden golden_settings_gc_10_1_2[] =
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0xffffffff, 0x00800000)
 };
 
-static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
+static bool gfx_v10_is_rlcg_rw(struct amdgpu_device *adev, u32 offset, uint32_t *flag, bool write)
+{
+	/* always programed by rlcg, only for gc */
+	if (offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI) ||
+	    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO) ||
+	    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH) ||
+	    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL) ||
+	    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX) ||
+	    offset == SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)) {
+		if (!amdgpu_sriov_reg_indirect_gc(adev))
+			*flag = GFX_RLCG_GC_WRITE_OLD;
+		else
+			*flag = write ? GFX_RLCG_GC_WRITE : GFX_RLCG_GC_READ;
+
+		return true;
+	}
+
+	/* currently support gc read/write, mmhub write */
+	if (offset >= SOC15_REG_OFFSET(GC, 0, mmSDMA0_DEC_START) &&
+	    offset <= SOC15_REG_OFFSET(GC, 0, mmRLC_GTS_OFFSET_MSB)) {
+		if (amdgpu_sriov_reg_indirect_gc(adev))
+			*flag = write ? GFX_RLCG_GC_WRITE : GFX_RLCG_GC_READ;
+		else
+			return false;
+	} else {
+		if (amdgpu_sriov_reg_indirect_mmhub(adev))
+			*flag = GFX_RLCG_MMHUB_WRITE;
+		else
+			return false;
+	}
+
+	return true;
+}
+
+static u32 gfx_v10_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 v, uint32_t flag)
 {
 	static void *scratch_reg0;
 	static void *scratch_reg1;
+	static void *scratch_reg2;
+	static void *scratch_reg3;
 	static void *spare_int;
+	static uint32_t grbm_cntl;
+	static uint32_t grbm_idx;
 	uint32_t i = 0;
 	uint32_t retries = 50000;
+	u32 ret = 0;
+
+	scratch_reg0 = adev->rmmio +
+		       (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0) * 4;
+	scratch_reg1 = adev->rmmio +
+		       (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1) * 4;
+	scratch_reg2 = adev->rmmio +
+		       (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG2) * 4;
+	scratch_reg3 = adev->rmmio +
+		       (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3) * 4;
+	spare_int = adev->rmmio +
+		    (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT) * 4;
+
+	grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
+	grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
+
+	if (offset == grbm_cntl || offset == grbm_idx) {
+		if (offset  == grbm_cntl)
+			writel(v, scratch_reg2);
+		else if (offset == grbm_idx)
+			writel(v, scratch_reg3);
+
+		writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
+	} else {
+		writel(v, scratch_reg0);
+		writel(offset | flag, scratch_reg1);
+		writel(1, spare_int);
+		for (i = 0; i < retries; i++) {
+			u32 tmp;
+
+			tmp = readl(scratch_reg1);
+			if (!(tmp & flag))
+				break;
 
-	scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
-	scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
-	spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
+			udelay(10);
+		}
 
-	if (amdgpu_sriov_runtime(adev)) {
-		pr_err("shouldn't call rlcg write register during runtime\n");
-		return;
+		if (i >= retries)
+			pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
 	}
 
-	writel(v, scratch_reg0);
-	writel(offset | 0x80000000, scratch_reg1);
-	writel(1, spare_int);
-	for (i = 0; i < retries; i++) {
-		u32 tmp;
+	ret = readl(scratch_reg0);
 
-		tmp = readl(scratch_reg1);
-		if (!(tmp & 0x80000000))
-			break;
+	return ret;
+}
 
-		udelay(10);
+static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 value, u32 flag)
+{
+	uint32_t rlcg_flag;
+
+	if (amdgpu_sriov_fullaccess(adev) &&
+	    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 1)) {
+		gfx_v10_rlcg_rw(adev, offset, value, rlcg_flag);
+
+		return;
 	}
+	if (flag & AMDGPU_REGS_NO_KIQ)
+		WREG32_NO_KIQ(offset, value);
+	else
+		WREG32(offset, value);
+}
+
+static u32 gfx_v10_rlcg_rreg(struct amdgpu_device *adev, u32 offset, u32 flag)
+{
+	uint32_t rlcg_flag;
 
-	if (i >= retries)
-		pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
+	if (amdgpu_sriov_fullaccess(adev) &&
+	    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 0))
+		return gfx_v10_rlcg_rw(adev, offset, 0, rlcg_flag);
+
+	if (flag & AMDGPU_REGS_NO_KIQ)
+		return RREG32_NO_KIQ(offset);
+	else
+		return RREG32(offset);
+
+	return 0;
 }
 
 static const struct soc15_reg_golden golden_settings_gc_10_1_nv14[] =
@@ -7888,6 +7982,7 @@ static const struct amdgpu_rlc_funcs gfx_v10_0_rlc_funcs_sriov = {
 	.start = gfx_v10_0_rlc_start,
 	.update_spm_vmid = gfx_v10_0_update_spm_vmid,
 	.rlcg_wreg = gfx_v10_rlcg_wreg,
+	.rlcg_rreg = gfx_v10_rlcg_rreg,
 	.is_rlcg_access_range = gfx_v10_0_is_rlcg_access_range,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 99f58439f3d5..6e49b239087a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -734,7 +734,7 @@ static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
 	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
 };
 
-static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
+void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
 {
 	static void *scratch_reg0;
 	static void *scratch_reg1;
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 8cdf5d1685cb..14bd794bbea6 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -77,27 +77,11 @@
 })
 
 #define WREG32_RLC(reg, value) \
-	do {							\
-		if (amdgpu_sriov_fullaccess(adev)) {    \
-			uint32_t i = 0;	\
-			uint32_t retries = 50000;	\
-			uint32_t r0 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0;	\
-			uint32_t r1 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1;	\
-			uint32_t spare_int = adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT;	\
-			WREG32(r0, value);	\
-			WREG32(r1, (reg | 0x80000000));	\
-			WREG32(spare_int, 0x1);	\
-			for (i = 0; i < retries; i++) {	\
-				u32 tmp = RREG32(r1);	\
-				if (!(tmp & 0x80000000))	\
-					break;	\
-				udelay(10);	\
-			}	\
-			if (i >= retries)	\
-				pr_err("timeout: rlcg program reg:0x%05x failed !\n", reg);	\
-		} else {	\
-			WREG32(reg, value); \
-		}	\
+	do { \
+		if (adev->gfx.rlc.funcs->rlcg_wreg) \
+			adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value, 0); \
+		else \
+			WREG32(reg, value);	\
 	} while (0)
 
 #define WREG32_RLC_EX(prefix, reg, value) \
@@ -125,23 +109,24 @@
 	} while (0)
 
 #define WREG32_SOC15_RLC_SHADOW(ip, inst, reg, value) \
-	do {							\
-		uint32_t target_reg = adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
-		if (amdgpu_sriov_fullaccess(adev)) {    \
-			uint32_t r2 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2;	\
-			uint32_t r3 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3;	\
-			uint32_t grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;   \
-			uint32_t grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;   \
-			if (target_reg == grbm_cntl) \
-				WREG32(r2, value);	\
-			else if (target_reg == grbm_idx) \
-				WREG32(r3, value);	\
-			WREG32(target_reg, value);	\
-		} else {	\
-			WREG32(target_reg, value); \
-		}	\
+	WREG32_RLC((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg), value)
+
+#define RREG32_RLC(reg) \
+	(adev->gfx.rlc.funcs->rlcg_rreg ? \
+		adev->gfx.rlc.funcs->rlcg_rreg(adev, reg, 0) : RREG32(reg))
+
+#define WREG32_RLC_NO_KIQ(reg, value) \
+	do { \
+		if (adev->gfx.rlc.funcs->rlcg_wreg) \
+			adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value, AMDGPU_REGS_NO_KIQ); \
+		else \
+			WREG32_NO_KIQ(reg, value);	\
 	} while (0)
 
+#define RREG32_RLC_NO_KIQ(reg) \
+	(adev->gfx.rlc.funcs->rlcg_rreg ? \
+		adev->gfx.rlc.funcs->rlcg_rreg(adev, reg, AMDGPU_REGS_NO_KIQ) : RREG32_NO_KIQ(reg))
+
 #define WREG32_SOC15_RLC_SHADOW_EX(prefix, ip, inst, reg, value) \
 	do {							\
 		uint32_t target_reg = adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
@@ -160,10 +145,13 @@
 		}	\
 	} while (0)
 
+#define RREG32_SOC15_RLC(ip, inst, reg) \
+	RREG32_RLC(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
+
 #define WREG32_SOC15_RLC(ip, inst, reg, value) \
 	do {							\
-			uint32_t target_reg = adev->reg_offset[GC_HWIP][0][reg##_BASE_IDX] + reg;\
-			WREG32_RLC(target_reg, value); \
+		uint32_t target_reg = adev->reg_offset[ip##_HWIP][0][reg##_BASE_IDX] + reg;\
+		WREG32_RLC(target_reg, value); \
 	} while (0)
 
 #define WREG32_SOC15_RLC_EX(prefix, ip, inst, reg, value) \
@@ -173,11 +161,14 @@
 	} while (0)
 
 #define WREG32_FIELD15_RLC(ip, idx, reg, field, val)   \
-    WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
-    (RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
-    & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
+	WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
+	(RREG32_RLC(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
+	& ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
 
 #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
-    WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset), value)
+	WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset), value)
+
+#define RREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset) \
+	RREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset))
 
 #endif
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov

[Deng, Emily]

[AMD Official Use Only - Internal Distribution Only]

Series Reviewed-by: Emily.Deng <Emily.Deng@amd.com>

>-----Original Message-----
>From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Peng
>Ju Zhou
>Sent: Wednesday, March 31, 2021 1:20 PM
>To: amd-gfx@lists.freedesktop.org
>Cc: Zhao, Jiange <Jiange.Zhao@amd.com>
>Subject: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov
>
>1. expand rlcg interface for gc & mmhub indirect access 2. add rlcg interface
>for no kiq
>
>Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
>---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   2 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h    |   3 +-
> drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 131 ++++++++++++++++++---
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      |   2 +-
> drivers/gpu/drm/amd/amdgpu/soc15_common.h  |  75 ++++++------
> 5 files changed, 150 insertions(+), 63 deletions(-)
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>index 060d0ae99453..438e2f732377 100644
>--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>@@ -490,7 +490,7 @@ void amdgpu_mm_wreg_mmio_rlc(struct
>amdgpu_device *adev,
>     adev->gfx.rlc.funcs &&
>     adev->gfx.rlc.funcs->is_rlcg_access_range) {
> if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
>-return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
>+return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0);
> } else {
> writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
> }
>diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>index aeaaae713c59..4fc2ce8ce8ab 100644
>--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>@@ -127,7 +127,8 @@ struct amdgpu_rlc_funcs {
> void (*reset)(struct amdgpu_device *adev);
> void (*start)(struct amdgpu_device *adev);
> void (*update_spm_vmid)(struct amdgpu_device *adev, unsigned
>vmid);
>-void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v);
>+void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v, u32
>flag);
>+u32 (*rlcg_rreg)(struct amdgpu_device *adev, u32 offset, u32 flag);
> bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t
>reg);  };
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>index b4fd0394cd08..85a6a10e048f 100644
>--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>@@ -177,6 +177,11 @@
> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid              0x2030
> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid_BASE_IDX     0
>
>+#define GFX_RLCG_GC_WRITE_OLD(0x8 << 28)
>+#define GFX_RLCG_GC_WRITE(0x0 << 28)
>+#define GFX_RLCG_GC_READ(0x1 << 28)
>+#define GFX_RLCG_MMHUB_WRITE(0x2 << 28)
>+
> MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
> MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
> MODULE_FIRMWARE("amdgpu/navi10_me.bin");
>@@ -1422,38 +1427,127 @@ static const struct soc15_reg_golden
>golden_settings_gc_10_1_2[] =
> SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0xffffffff,
>0x00800000)  };
>
>-static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
>+static bool gfx_v10_is_rlcg_rw(struct amdgpu_device *adev, u32 offset,
>+uint32_t *flag, bool write) {
>+/* always programed by rlcg, only for gc */
>+if (offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)) {
>+if (!amdgpu_sriov_reg_indirect_gc(adev))
>+*flag = GFX_RLCG_GC_WRITE_OLD;
>+else
>+*flag = write ? GFX_RLCG_GC_WRITE :
>GFX_RLCG_GC_READ;
>+
>+return true;
>+}
>+
>+/* currently support gc read/write, mmhub write */
>+if (offset >= SOC15_REG_OFFSET(GC, 0, mmSDMA0_DEC_START) &&
>+    offset <= SOC15_REG_OFFSET(GC, 0, mmRLC_GTS_OFFSET_MSB)) {
>+if (amdgpu_sriov_reg_indirect_gc(adev))
>+*flag = write ? GFX_RLCG_GC_WRITE :
>GFX_RLCG_GC_READ;
>+else
>+return false;
>+} else {
>+if (amdgpu_sriov_reg_indirect_mmhub(adev))
>+*flag = GFX_RLCG_MMHUB_WRITE;
>+else
>+return false;
>+}
>+
>+return true;
>+}
>+
>+static u32 gfx_v10_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32
>+v, uint32_t flag)
> {
> static void *scratch_reg0;
> static void *scratch_reg1;
>+static void *scratch_reg2;
>+static void *scratch_reg3;
> static void *spare_int;
>+static uint32_t grbm_cntl;
>+static uint32_t grbm_idx;
> uint32_t i = 0;
> uint32_t retries = 50000;
>+u32 ret = 0;
>+
>+scratch_reg0 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0) * 4;
>+scratch_reg1 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1) * 4;
>+scratch_reg2 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG2) * 4;
>+scratch_reg3 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG3) * 4;
>+spare_int = adev->rmmio +
>+    (adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>+mmRLC_SPARE_INT) * 4;
>+
>+grbm_cntl = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>mmGRBM_GFX_CNTL;
>+grbm_idx = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>+mmGRBM_GFX_INDEX;
>+
>+if (offset == grbm_cntl || offset == grbm_idx) {
>+if (offset  == grbm_cntl)
>+writel(v, scratch_reg2);
>+else if (offset == grbm_idx)
>+writel(v, scratch_reg3);
>+
>+writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
>+} else {
>+writel(v, scratch_reg0);
>+writel(offset | flag, scratch_reg1);
>+writel(1, spare_int);
>+for (i = 0; i < retries; i++) {
>+u32 tmp;
>+
>+tmp = readl(scratch_reg1);
>+if (!(tmp & flag))
>+break;
>
>-scratch_reg0 = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0)*4;
>-scratch_reg1 = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1)*4;
>-spare_int = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>mmRLC_SPARE_INT)*4;
>+udelay(10);
>+}
>
>-if (amdgpu_sriov_runtime(adev)) {
>-pr_err("shouldn't call rlcg write register during runtime\n");
>-return;
>+if (i >= retries)
>+pr_err("timeout: rlcg program reg:0x%05x failed !\n",
>offset);
> }
>
>-writel(v, scratch_reg0);
>-writel(offset | 0x80000000, scratch_reg1);
>-writel(1, spare_int);
>-for (i = 0; i < retries; i++) {
>-u32 tmp;
>+ret = readl(scratch_reg0);
>
>-tmp = readl(scratch_reg1);
>-if (!(tmp & 0x80000000))
>-break;
>+return ret;
>+}
>
>-udelay(10);
>+static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset,
>+u32 value, u32 flag) {
>+uint32_t rlcg_flag;
>+
>+if (amdgpu_sriov_fullaccess(adev) &&
>+    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 1)) {
>+gfx_v10_rlcg_rw(adev, offset, value, rlcg_flag);
>+
>+return;
> }
>+if (flag & AMDGPU_REGS_NO_KIQ)
>+WREG32_NO_KIQ(offset, value);
>+else
>+WREG32(offset, value);
>+}
>+
>+static u32 gfx_v10_rlcg_rreg(struct amdgpu_device *adev, u32 offset,
>+u32 flag) {
>+uint32_t rlcg_flag;
>
>-if (i >= retries)
>-pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
>+if (amdgpu_sriov_fullaccess(adev) &&
>+    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 0))
>+return gfx_v10_rlcg_rw(adev, offset, 0, rlcg_flag);
>+
>+if (flag & AMDGPU_REGS_NO_KIQ)
>+return RREG32_NO_KIQ(offset);
>+else
>+return RREG32(offset);
>+
>+return 0;
> }
>
> static const struct soc15_reg_golden golden_settings_gc_10_1_nv14[] = @@ -
>7888,6 +7982,7 @@ static const struct amdgpu_rlc_funcs
>gfx_v10_0_rlc_funcs_sriov = {
> .start = gfx_v10_0_rlc_start,
> .update_spm_vmid = gfx_v10_0_update_spm_vmid,
> .rlcg_wreg = gfx_v10_rlcg_wreg,
>+.rlcg_rreg = gfx_v10_rlcg_rreg,
> .is_rlcg_access_range = gfx_v10_0_is_rlcg_access_range,  };
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>index 99f58439f3d5..6e49b239087a 100644
>--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>@@ -734,7 +734,7 @@ static const u32
>GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
> mmRLC_SRM_INDEX_CNTL_DATA_7 -
>mmRLC_SRM_INDEX_CNTL_DATA_0,  };
>
>-static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
>+void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v,
>+u32 flag)
> {
> static void *scratch_reg0;
> static void *scratch_reg1;
>diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>index 8cdf5d1685cb..14bd794bbea6 100644
>--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>@@ -77,27 +77,11 @@
> })
>
> #define WREG32_RLC(reg, value) \
>-do {\
>-if (amdgpu_sriov_fullaccess(adev)) {    \
>-uint32_t i = 0;\
>-uint32_t retries = 50000;\
>-uint32_t r0 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0;\
>-uint32_t r1 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1;\
>-uint32_t spare_int = adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>mmRLC_SPARE_INT;\
>-WREG32(r0, value);\
>-WREG32(r1, (reg | 0x80000000));\
>-WREG32(spare_int, 0x1);\
>-for (i = 0; i < retries; i++) {\
>-u32 tmp = RREG32(r1);\
>-if (!(tmp & 0x80000000))\
>-break;\
>-udelay(10);\
>-}\
>-if (i >= retries)\
>-pr_err("timeout: rlcg program reg:0x%05x
>failed !\n", reg);\
>-} else {\
>-WREG32(reg, value); \
>-}\
>+do { \
>+if (adev->gfx.rlc.funcs->rlcg_wreg) \
>+adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value, 0); \
>+else \
>+WREG32(reg, value);\
> } while (0)
>
> #define WREG32_RLC_EX(prefix, reg, value) \ @@ -125,23 +109,24 @@
> } while (0)
>
> #define WREG32_SOC15_RLC_SHADOW(ip, inst, reg, value) \
>-do {\
>-uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
>-if (amdgpu_sriov_fullaccess(adev)) {    \
>-uint32_t r2 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG2;\
>-uint32_t r3 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG3;\
>-uint32_t grbm_cntl = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>mmGRBM_GFX_CNTL;   \
>-uint32_t grbm_idx = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>mmGRBM_GFX_INDEX;   \
>-if (target_reg == grbm_cntl) \
>-WREG32(r2, value);\
>-else if (target_reg == grbm_idx) \
>-WREG32(r3, value);\
>-WREG32(target_reg, value);\
>-} else {\
>-WREG32(target_reg, value); \
>-}\
>+WREG32_RLC((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg),
>+value)
>+
>+#define RREG32_RLC(reg) \
>+(adev->gfx.rlc.funcs->rlcg_rreg ? \
>+adev->gfx.rlc.funcs->rlcg_rreg(adev, reg, 0) : RREG32(reg))
>+
>+#define WREG32_RLC_NO_KIQ(reg, value) \
>+do { \
>+if (adev->gfx.rlc.funcs->rlcg_wreg) \
>+adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value,
>AMDGPU_REGS_NO_KIQ); \
>+else \
>+WREG32_NO_KIQ(reg, value);\
> } while (0)
>
>+#define RREG32_RLC_NO_KIQ(reg) \
>+(adev->gfx.rlc.funcs->rlcg_rreg ? \
>+adev->gfx.rlc.funcs->rlcg_rreg(adev, reg,
>AMDGPU_REGS_NO_KIQ) :
>+RREG32_NO_KIQ(reg))
>+
> #define WREG32_SOC15_RLC_SHADOW_EX(prefix, ip, inst, reg, value) \
> do {\
> uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\ @@ -160,10 +145,13
>@@
> }\
> } while (0)
>
>+#define RREG32_SOC15_RLC(ip, inst, reg) \
>+RREG32_RLC(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>+
> #define WREG32_SOC15_RLC(ip, inst, reg, value) \
> do {\
>-uint32_t target_reg = adev-
>>reg_offset[GC_HWIP][0][reg##_BASE_IDX] + reg;\
>-WREG32_RLC(target_reg, value); \
>+uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][0][reg##_BASE_IDX] + reg;\
>+WREG32_RLC(target_reg, value); \
> } while (0)
>
> #define WREG32_SOC15_RLC_EX(prefix, ip, inst, reg, value) \ @@ -173,11
>+161,14 @@
> } while (0)
>
> #define WREG32_FIELD15_RLC(ip, idx, reg, field, val)   \
>-    WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>mm##reg), \
>-    (RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>mm##reg) \
>-    & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>+WREG32_RLC((adev-
>>reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
>+(RREG32_RLC(adev-
>>reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
>+& ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>
> #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
>-    WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
>+ offset), value)
>+WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>++ offset), value)
>+
>+#define RREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset) \
>+RREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>++ offset))
>
> #endif
>--
>2.17.1
>
>_______________________________________________
>amd-gfx mailing list
>amd-gfx@lists.freedesktop.org
>https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.fr
>eedesktop.org%2Fmailman%2Flistinfo%2Famd-
>gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C5418b86bcb4042c1a
>b1c08d8f404c823%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C63
>7527648681476268%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAi
>LCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=SIC
>r%2FbhwrejJzpuhJmsnJiu6TP%2Fvu075po7%2BJ70Foec%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov

[Chen, Guchun]

[AMD Public Use]

Hi Peng Ju,

Patch 4 breaks the driver modprobe sequence for the ASICs with GFX IP v9.0. The modification in WREG32_RLC will route to one different path for GFX v9. Please check it.

Regards,
Guchun

-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Deng, Emily
Sent: Thursday, April 1, 2021 2:01 PM
To: Zhou, Peng Ju <PengJu.Zhou@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Zhao, Jiange <Jiange.Zhao@amd.com>
Subject: RE: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov

[AMD Official Use Only - Internal Distribution Only]

[AMD Official Use Only - Internal Distribution Only]

Series Reviewed-by: Emily.Deng <Emily.Deng@amd.com>

>-----Original Message-----
>From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Peng 
>Ju Zhou
>Sent: Wednesday, March 31, 2021 1:20 PM
>To: amd-gfx@lists.freedesktop.org
>Cc: Zhao, Jiange <Jiange.Zhao@amd.com>
>Subject: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 
>sriov
>
>1. expand rlcg interface for gc & mmhub indirect access 2. add rlcg 
>interface for no kiq
>
>Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
>---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   2 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h    |   3 +-
> drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 131 ++++++++++++++++++---
> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      |   2 +-
> drivers/gpu/drm/amd/amdgpu/soc15_common.h  |  75 ++++++------
> 5 files changed, 150 insertions(+), 63 deletions(-)
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>index 060d0ae99453..438e2f732377 100644
>--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>@@ -490,7 +490,7 @@ void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device 
>*adev,
>     adev->gfx.rlc.funcs &&
>     adev->gfx.rlc.funcs->is_rlcg_access_range) {  if 
>(adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) -return 
>adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
>+return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0);
> } else {
> writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  } diff --git 
>a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>index aeaaae713c59..4fc2ce8ce8ab 100644
>--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>@@ -127,7 +127,8 @@ struct amdgpu_rlc_funcs {  void (*reset)(struct 
>amdgpu_device *adev);  void (*start)(struct amdgpu_device *adev);  void 
>(*update_spm_vmid)(struct amdgpu_device *adev, unsigned vmid); -void 
>(*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v);
>+void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v, u32
>flag);
>+u32 (*rlcg_rreg)(struct amdgpu_device *adev, u32 offset, u32 flag);
> bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t 
>reg);  };
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>index b4fd0394cd08..85a6a10e048f 100644
>--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>@@ -177,6 +177,11 @@
> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid              0x2030
> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid_BASE_IDX     0
>
>+#define GFX_RLCG_GC_WRITE_OLD(0x8 << 28) #define GFX_RLCG_GC_WRITE(0x0 
>+<< 28) #define GFX_RLCG_GC_READ(0x1 << 28) #define 
>+GFX_RLCG_MMHUB_WRITE(0x2 << 28)
>+
> MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
> MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
> MODULE_FIRMWARE("amdgpu/navi10_me.bin");
>@@ -1422,38 +1427,127 @@ static const struct soc15_reg_golden 
>golden_settings_gc_10_1_2[] =  SOC15_REG_GOLDEN_VALUE(GC, 0, 
>mmUTCL1_CTRL, 0xffffffff,
>0x00800000)  };
>
>-static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset, 
>u32 v)
>+static bool gfx_v10_is_rlcg_rw(struct amdgpu_device *adev, u32 offset, 
>+uint32_t *flag, bool write) {
>+/* always programed by rlcg, only for gc */ if (offset == 
>+SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX) ||
>+    offset == SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)) { if 
>+(!amdgpu_sriov_reg_indirect_gc(adev))
>+*flag = GFX_RLCG_GC_WRITE_OLD;
>+else
>+*flag = write ? GFX_RLCG_GC_WRITE :
>GFX_RLCG_GC_READ;
>+
>+return true;
>+}
>+
>+/* currently support gc read/write, mmhub write */ if (offset >= 
>+SOC15_REG_OFFSET(GC, 0, mmSDMA0_DEC_START) &&
>+    offset <= SOC15_REG_OFFSET(GC, 0, mmRLC_GTS_OFFSET_MSB)) { if 
>+(amdgpu_sriov_reg_indirect_gc(adev))
>+*flag = write ? GFX_RLCG_GC_WRITE :
>GFX_RLCG_GC_READ;
>+else
>+return false;
>+} else {
>+if (amdgpu_sriov_reg_indirect_mmhub(adev))
>+*flag = GFX_RLCG_MMHUB_WRITE;
>+else
>+return false;
>+}
>+
>+return true;
>+}
>+
>+static u32 gfx_v10_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 
>+v, uint32_t flag)
> {
> static void *scratch_reg0;
> static void *scratch_reg1;
>+static void *scratch_reg2;
>+static void *scratch_reg3;
> static void *spare_int;
>+static uint32_t grbm_cntl;
>+static uint32_t grbm_idx;
> uint32_t i = 0;
> uint32_t retries = 50000;
>+u32 ret = 0;
>+
>+scratch_reg0 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0) * 4;
>+scratch_reg1 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1) * 4;
>+scratch_reg2 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG2) * 4;
>+scratch_reg3 = adev->rmmio +
>+       (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG3) * 4;
>+spare_int = adev->rmmio +
>+    (adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>+mmRLC_SPARE_INT) * 4;
>+
>+grbm_cntl = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>mmGRBM_GFX_CNTL;
>+grbm_idx = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>+mmGRBM_GFX_INDEX;
>+
>+if (offset == grbm_cntl || offset == grbm_idx) { if (offset  == 
>+grbm_cntl) writel(v, scratch_reg2); else if (offset == grbm_idx) 
>+writel(v, scratch_reg3);
>+
>+writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); } else { 
>+writel(v, scratch_reg0); writel(offset | flag, scratch_reg1); 
>+writel(1, spare_int); for (i = 0; i < retries; i++) {
>+u32 tmp;
>+
>+tmp = readl(scratch_reg1);
>+if (!(tmp & flag))
>+break;
>
>-scratch_reg0 = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0)*4;
>-scratch_reg1 = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1)*4;
>-spare_int = adev->rmmio + (adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>mmRLC_SPARE_INT)*4;
>+udelay(10);
>+}
>
>-if (amdgpu_sriov_runtime(adev)) {
>-pr_err("shouldn't call rlcg write register during runtime\n"); 
>-return;
>+if (i >= retries)
>+pr_err("timeout: rlcg program reg:0x%05x failed !\n",
>offset);
> }
>
>-writel(v, scratch_reg0);
>-writel(offset | 0x80000000, scratch_reg1); -writel(1, spare_int); -for 
>(i = 0; i < retries; i++) {
>-u32 tmp;
>+ret = readl(scratch_reg0);
>
>-tmp = readl(scratch_reg1);
>-if (!(tmp & 0x80000000))
>-break;
>+return ret;
>+}
>
>-udelay(10);
>+static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset,
>+u32 value, u32 flag) {
>+uint32_t rlcg_flag;
>+
>+if (amdgpu_sriov_fullaccess(adev) &&
>+    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 1)) { 
>+gfx_v10_rlcg_rw(adev, offset, value, rlcg_flag);
>+
>+return;
> }
>+if (flag & AMDGPU_REGS_NO_KIQ)
>+WREG32_NO_KIQ(offset, value);
>+else
>+WREG32(offset, value);
>+}
>+
>+static u32 gfx_v10_rlcg_rreg(struct amdgpu_device *adev, u32 offset,
>+u32 flag) {
>+uint32_t rlcg_flag;
>
>-if (i >= retries)
>-pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
>+if (amdgpu_sriov_fullaccess(adev) &&
>+    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 0)) return 
>+gfx_v10_rlcg_rw(adev, offset, 0, rlcg_flag);
>+
>+if (flag & AMDGPU_REGS_NO_KIQ)
>+return RREG32_NO_KIQ(offset);
>+else
>+return RREG32(offset);
>+
>+return 0;
> }
>
> static const struct soc15_reg_golden golden_settings_gc_10_1_nv14[] = 
>@@ -
>7888,6 +7982,7 @@ static const struct amdgpu_rlc_funcs 
>gfx_v10_0_rlc_funcs_sriov = {  .start = gfx_v10_0_rlc_start,  
>.update_spm_vmid = gfx_v10_0_update_spm_vmid,  .rlcg_wreg = 
>gfx_v10_rlcg_wreg,
>+.rlcg_rreg = gfx_v10_rlcg_rreg,
> .is_rlcg_access_range = gfx_v10_0_is_rlcg_access_range,  };
>
>diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>index 99f58439f3d5..6e49b239087a 100644
>--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>@@ -734,7 +734,7 @@ static const u32
>GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
> mmRLC_SRM_INDEX_CNTL_DATA_7 -
>mmRLC_SRM_INDEX_CNTL_DATA_0,  };
>
>-static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, 
>u32 v)
>+void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v,
>+u32 flag)
> {
> static void *scratch_reg0;
> static void *scratch_reg1;
>diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>index 8cdf5d1685cb..14bd794bbea6 100644
>--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>@@ -77,27 +77,11 @@
> })
>
> #define WREG32_RLC(reg, value) \
>-do {\
>-if (amdgpu_sriov_fullaccess(adev)) {    \
>-uint32_t i = 0;\
>-uint32_t retries = 50000;\
>-uint32_t r0 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>mmSCRATCH_REG0;\
>-uint32_t r1 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG1;\
>-uint32_t spare_int = adev-
>>reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>mmRLC_SPARE_INT;\
>-WREG32(r0, value);\
>-WREG32(r1, (reg | 0x80000000));\
>-WREG32(spare_int, 0x1);\
>-for (i = 0; i < retries; i++) {\
>-u32 tmp = RREG32(r1);\
>-if (!(tmp & 0x80000000))\
>-break;\
>-udelay(10);\
>-}\
>-if (i >= retries)\
>-pr_err("timeout: rlcg program reg:0x%05x failed !\n", reg);\ -} else 
>{\ -WREG32(reg, value); \ -}\
>+do { \
>+if (adev->gfx.rlc.funcs->rlcg_wreg) \
>+adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value, 0); \
>+else \
>+WREG32(reg, value);\
> } while (0)
>
> #define WREG32_RLC_EX(prefix, reg, value) \ @@ -125,23 +109,24 @@ } 
> while (0)
>
> #define WREG32_SOC15_RLC_SHADOW(ip, inst, reg, value) \ -do {\ 
>-uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
>-if (amdgpu_sriov_fullaccess(adev)) {    \
>-uint32_t r2 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG2;\
>-uint32_t r3 = adev-
>>reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>mmSCRATCH_REG3;\
>-uint32_t grbm_cntl = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>mmGRBM_GFX_CNTL;   \
>-uint32_t grbm_idx = adev-
>>reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>mmGRBM_GFX_INDEX;   \
>-if (target_reg == grbm_cntl) \
>-WREG32(r2, value);\
>-else if (target_reg == grbm_idx) \
>-WREG32(r3, value);\
>-WREG32(target_reg, value);\
>-} else {\
>-WREG32(target_reg, value); \
>-}\
>+WREG32_RLC((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg),
>+value)
>+
>+#define RREG32_RLC(reg) \
>+(adev->gfx.rlc.funcs->rlcg_rreg ? \
>+adev->gfx.rlc.funcs->rlcg_rreg(adev, reg, 0) : RREG32(reg))
>+
>+#define WREG32_RLC_NO_KIQ(reg, value) \ do { \ if 
>+(adev->gfx.rlc.funcs->rlcg_wreg) \
>+adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value,
>AMDGPU_REGS_NO_KIQ); \
>+else \
>+WREG32_NO_KIQ(reg, value);\
> } while (0)
>
>+#define RREG32_RLC_NO_KIQ(reg) \
>+(adev->gfx.rlc.funcs->rlcg_rreg ? \
>+adev->gfx.rlc.funcs->rlcg_rreg(adev, reg,
>AMDGPU_REGS_NO_KIQ) :
>+RREG32_NO_KIQ(reg))
>+
> #define WREG32_SOC15_RLC_SHADOW_EX(prefix, ip, inst, reg, value) \ do 
> {\ uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\ @@ -160,10 +145,13
>@@
> }\
> } while (0)
>
>+#define RREG32_SOC15_RLC(ip, inst, reg) \ 
>+RREG32_RLC(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>+
> #define WREG32_SOC15_RLC(ip, inst, reg, value) \  do {\ -uint32_t 
>target_reg = adev-
>>reg_offset[GC_HWIP][0][reg##_BASE_IDX] + reg;\
>-WREG32_RLC(target_reg, value); \
>+uint32_t target_reg = adev-
>>reg_offset[ip##_HWIP][0][reg##_BASE_IDX] + reg;\
>+WREG32_RLC(target_reg, value); \
> } while (0)
>
> #define WREG32_SOC15_RLC_EX(prefix, ip, inst, reg, value) \ @@ -173,11
>+161,14 @@
> } while (0)
>
> #define WREG32_FIELD15_RLC(ip, idx, reg, field, val)   \
>-    WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>mm##reg), \
>-    (RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>mm##reg) \
>-    & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>+WREG32_RLC((adev-
>>reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
>+(RREG32_RLC(adev-
>>reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
>+& ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>
> #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
>-    WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
>+ offset), value)
>+WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>++ offset), value)
>+
>+#define RREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset) \ 
>+RREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>reg)
>++ offset))
>
> #endif
>--
>2.17.1
>
>_______________________________________________
>amd-gfx mailing list
>amd-gfx@lists.freedesktop.org
>https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists
>.fr
>eedesktop.org%2Fmailman%2Flistinfo%2Famd-
>gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C5418b86bcb4042c1a
>b1c08d8f404c823%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C63
>7527648681476268%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAi
>LCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=SIC
>r%2FbhwrejJzpuhJmsnJiu6TP%2Fvu075po7%2BJ70Foec%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=04%7C01%7Cguchun.chen%40amd.com%7C1506b6df5d124f715d3e08d8f4d37f4b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637528536512044976%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=rxuB%2F5If9Ij%2F03h9VxU2tJcB4FdslLePEYSGu1jet%2Fs%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov

[Felix Kuehling]

Several people here ran into the same problem, including myself. Please 
revert the patch until you can come up with a fix. See log messages below.

Thanks,
   Felix


[   19.760590] amdgpu: timeout: rlcg program reg:0x02984 failed !
[   20.306299] amdgpu: timeout: rlcg program reg:0x02983 failed !
[   20.851868] amdgpu: timeout: rlcg program reg:0x02982 failed !
[   21.397519] amdgpu: timeout: rlcg program reg:0x02985 failed !
[   21.942955] amdgpu: timeout: rlcg program reg:0x02986 failed !
[   22.488260] amdgpu: timeout: rlcg program reg:0x02987 failed !
[   23.033506] amdgpu: timeout: rlcg program reg:0x02840 failed !
[   23.578778] amdgpu: timeout: rlcg program reg:0x02841 failed !
[   24.123999] amdgpu: timeout: rlcg program reg:0x02842 failed !
[   24.669278] amdgpu: timeout: rlcg program reg:0x02857 failed !
[   24.684331] amdgpu: hwmgr_sw_init smu backed is vega10_smu
[   25.465325] amdgpu: timeout: rlcg program reg:0x0eca3 failed !
[   26.011164] amdgpu: timeout: rlcg program reg:0x0eca2 failed !
[   26.556592] amdgpu: timeout: rlcg program reg:0x0eca4 failed !
[   26.765861] amdgpu 0000:08:00.0: [drm:amdgpu_ring_test_helper [amdgpu]] *ERROR* ring kiq_2.1.0 test failed (-110)
[   26.776337] [drm:amdgpu_gfx_enable_kcq [amdgpu]] *ERROR* KCQ enable failed
[   26.783359] [drm:amdgpu_device_init [amdgpu]] *ERROR* hw_init of IP block <gfx_v9_0> failed -110
[   26.792324] amdgpu 0000:08:00.0: amdgpu: amdgpu_device_ip_init failed
[   26.798842] amdgpu 0000:08:00.0: amdgpu: Fatal error during GPU init
[   26.805649] amdgpu: probe of 0000:08:00.0 failed with error -110

On 2021-04-04 10:18 a.m., Chen, Guchun wrote:
> [AMD Public Use]
>
> Hi Peng Ju,
>
> Patch 4 breaks the driver modprobe sequence for the ASICs with GFX IP v9.0. The modification in WREG32_RLC will route to one different path for GFX v9. Please check it.
>
> Regards,
> Guchun
>
> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Deng, Emily
> Sent: Thursday, April 1, 2021 2:01 PM
> To: Zhou, Peng Ju <PengJu.Zhou@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Zhao, Jiange <Jiange.Zhao@amd.com>
> Subject: RE: [PATCH 4/4] drm/amdgpu: indirect register access for nv12 sriov
>
> [AMD Official Use Only - Internal Distribution Only]
>
> [AMD Official Use Only - Internal Distribution Only]
>
> Series Reviewed-by: Emily.Deng <Emily.Deng@amd.com>
>
>> -----Original Message-----
>> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Peng
>> Ju Zhou
>> Sent: Wednesday, March 31, 2021 1:20 PM
>> To: amd-gfx@lists.freedesktop.org
>> Cc: Zhao, Jiange <Jiange.Zhao@amd.com>
>> Subject: [PATCH 4/4] drm/amdgpu: indirect register access for nv12
>> sriov
>>
>> 1. expand rlcg interface for gc & mmhub indirect access 2. add rlcg
>> interface for no kiq
>>
>> Signed-off-by: Peng Ju Zhou <PengJu.Zhou@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   2 +-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h    |   3 +-
>> drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 131 ++++++++++++++++++---
>> drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      |   2 +-
>> drivers/gpu/drm/amd/amdgpu/soc15_common.h  |  75 ++++++------
>> 5 files changed, 150 insertions(+), 63 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 060d0ae99453..438e2f732377 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -490,7 +490,7 @@ void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device
>> *adev,
>>      adev->gfx.rlc.funcs &&
>>      adev->gfx.rlc.funcs->is_rlcg_access_range) {  if
>> (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) -return
>> adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
>> +return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0);
>> } else {
>> writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));  } diff --git
>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>> index aeaaae713c59..4fc2ce8ce8ab 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
>> @@ -127,7 +127,8 @@ struct amdgpu_rlc_funcs {  void (*reset)(struct
>> amdgpu_device *adev);  void (*start)(struct amdgpu_device *adev);  void
>> (*update_spm_vmid)(struct amdgpu_device *adev, unsigned vmid); -void
>> (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v);
>> +void (*rlcg_wreg)(struct amdgpu_device *adev, u32 offset, u32 v, u32
>> flag);
>> +u32 (*rlcg_rreg)(struct amdgpu_device *adev, u32 offset, u32 flag);
>> bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t
>> reg);  };
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>> b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>> index b4fd0394cd08..85a6a10e048f 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
>> @@ -177,6 +177,11 @@
>> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid              0x2030
>> #define mmGC_THROTTLE_CTRL_Sienna_Cichlid_BASE_IDX     0
>>
>> +#define GFX_RLCG_GC_WRITE_OLD(0x8 << 28) #define GFX_RLCG_GC_WRITE(0x0
>> +<< 28) #define GFX_RLCG_GC_READ(0x1 << 28) #define
>> +GFX_RLCG_MMHUB_WRITE(0x2 << 28)
>> +
>> MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
>> MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
>> MODULE_FIRMWARE("amdgpu/navi10_me.bin");
>> @@ -1422,38 +1427,127 @@ static const struct soc15_reg_golden
>> golden_settings_gc_10_1_2[] =  SOC15_REG_GOLDEN_VALUE(GC, 0,
>> mmUTCL1_CTRL, 0xffffffff,
>> 0x00800000)  };
>>
>> -static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset,
>> u32 v)
>> +static bool gfx_v10_is_rlcg_rw(struct amdgpu_device *adev, u32 offset,
>> +uint32_t *flag, bool write) {
>> +/* always programed by rlcg, only for gc */ if (offset ==
>> +SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI) ||
>> +    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO) ||
>> +    offset == SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH) ||
>> +    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL) ||
>> +    offset == SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX) ||
>> +    offset == SOC15_REG_OFFSET(GC, 0, mmCP_ME_CNTL)) { if
>> +(!amdgpu_sriov_reg_indirect_gc(adev))
>> +*flag = GFX_RLCG_GC_WRITE_OLD;
>> +else
>> +*flag = write ? GFX_RLCG_GC_WRITE :
>> GFX_RLCG_GC_READ;
>> +
>> +return true;
>> +}
>> +
>> +/* currently support gc read/write, mmhub write */ if (offset >=
>> +SOC15_REG_OFFSET(GC, 0, mmSDMA0_DEC_START) &&
>> +    offset <= SOC15_REG_OFFSET(GC, 0, mmRLC_GTS_OFFSET_MSB)) { if
>> +(amdgpu_sriov_reg_indirect_gc(adev))
>> +*flag = write ? GFX_RLCG_GC_WRITE :
>> GFX_RLCG_GC_READ;
>> +else
>> +return false;
>> +} else {
>> +if (amdgpu_sriov_reg_indirect_mmhub(adev))
>> +*flag = GFX_RLCG_MMHUB_WRITE;
>> +else
>> +return false;
>> +}
>> +
>> +return true;
>> +}
>> +
>> +static u32 gfx_v10_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32
>> +v, uint32_t flag)
>> {
>> static void *scratch_reg0;
>> static void *scratch_reg1;
>> +static void *scratch_reg2;
>> +static void *scratch_reg3;
>> static void *spare_int;
>> +static uint32_t grbm_cntl;
>> +static uint32_t grbm_idx;
>> uint32_t i = 0;
>> uint32_t retries = 50000;
>> +u32 ret = 0;
>> +
>> +scratch_reg0 = adev->rmmio +
>> +       (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>> mmSCRATCH_REG0) * 4;
>> +scratch_reg1 = adev->rmmio +
>> +       (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG1) * 4;
>> +scratch_reg2 = adev->rmmio +
>> +       (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>> mmSCRATCH_REG2) * 4;
>> +scratch_reg3 = adev->rmmio +
>> +       (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG3) * 4;
>> +spare_int = adev->rmmio +
>> +    (adev-
>>> reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>> +mmRLC_SPARE_INT) * 4;
>> +
>> +grbm_cntl = adev-
>>> reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>> mmGRBM_GFX_CNTL;
>> +grbm_idx = adev-
>>> reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>> +mmGRBM_GFX_INDEX;
>> +
>> +if (offset == grbm_cntl || offset == grbm_idx) { if (offset  ==
>> +grbm_cntl) writel(v, scratch_reg2); else if (offset == grbm_idx)
>> +writel(v, scratch_reg3);
>> +
>> +writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); } else {
>> +writel(v, scratch_reg0); writel(offset | flag, scratch_reg1);
>> +writel(1, spare_int); for (i = 0; i < retries; i++) {
>> +u32 tmp;
>> +
>> +tmp = readl(scratch_reg1);
>> +if (!(tmp & flag))
>> +break;
>>
>> -scratch_reg0 = adev->rmmio + (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>> mmSCRATCH_REG0)*4;
>> -scratch_reg1 = adev->rmmio + (adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG1)*4;
>> -spare_int = adev->rmmio + (adev-
>>> reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>> mmRLC_SPARE_INT)*4;
>> +udelay(10);
>> +}
>>
>> -if (amdgpu_sriov_runtime(adev)) {
>> -pr_err("shouldn't call rlcg write register during runtime\n");
>> -return;
>> +if (i >= retries)
>> +pr_err("timeout: rlcg program reg:0x%05x failed !\n",
>> offset);
>> }
>>
>> -writel(v, scratch_reg0);
>> -writel(offset | 0x80000000, scratch_reg1); -writel(1, spare_int); -for
>> (i = 0; i < retries; i++) {
>> -u32 tmp;
>> +ret = readl(scratch_reg0);
>>
>> -tmp = readl(scratch_reg1);
>> -if (!(tmp & 0x80000000))
>> -break;
>> +return ret;
>> +}
>>
>> -udelay(10);
>> +static void gfx_v10_rlcg_wreg(struct amdgpu_device *adev, u32 offset,
>> +u32 value, u32 flag) {
>> +uint32_t rlcg_flag;
>> +
>> +if (amdgpu_sriov_fullaccess(adev) &&
>> +    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 1)) {
>> +gfx_v10_rlcg_rw(adev, offset, value, rlcg_flag);
>> +
>> +return;
>> }
>> +if (flag & AMDGPU_REGS_NO_KIQ)
>> +WREG32_NO_KIQ(offset, value);
>> +else
>> +WREG32(offset, value);
>> +}
>> +
>> +static u32 gfx_v10_rlcg_rreg(struct amdgpu_device *adev, u32 offset,
>> +u32 flag) {
>> +uint32_t rlcg_flag;
>>
>> -if (i >= retries)
>> -pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
>> +if (amdgpu_sriov_fullaccess(adev) &&
>> +    gfx_v10_is_rlcg_rw(adev, offset, &rlcg_flag, 0)) return
>> +gfx_v10_rlcg_rw(adev, offset, 0, rlcg_flag);
>> +
>> +if (flag & AMDGPU_REGS_NO_KIQ)
>> +return RREG32_NO_KIQ(offset);
>> +else
>> +return RREG32(offset);
>> +
>> +return 0;
>> }
>>
>> static const struct soc15_reg_golden golden_settings_gc_10_1_nv14[] =
>> @@ -
>> 7888,6 +7982,7 @@ static const struct amdgpu_rlc_funcs
>> gfx_v10_0_rlc_funcs_sriov = {  .start = gfx_v10_0_rlc_start,
>> .update_spm_vmid = gfx_v10_0_update_spm_vmid,  .rlcg_wreg =
>> gfx_v10_rlcg_wreg,
>> +.rlcg_rreg = gfx_v10_rlcg_rreg,
>> .is_rlcg_access_range = gfx_v10_0_is_rlcg_access_range,  };
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> index 99f58439f3d5..6e49b239087a 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
>> @@ -734,7 +734,7 @@ static const u32
>> GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
>> mmRLC_SRM_INDEX_CNTL_DATA_7 -
>> mmRLC_SRM_INDEX_CNTL_DATA_0,  };
>>
>> -static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset,
>> u32 v)
>> +void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v,
>> +u32 flag)
>> {
>> static void *scratch_reg0;
>> static void *scratch_reg1;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>> b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>> index 8cdf5d1685cb..14bd794bbea6 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
>> @@ -77,27 +77,11 @@
>> })
>>
>> #define WREG32_RLC(reg, value) \
>> -do {\
>> -if (amdgpu_sriov_fullaccess(adev)) {    \
>> -uint32_t i = 0;\
>> -uint32_t retries = 50000;\
>> -uint32_t r0 = adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] +
>> mmSCRATCH_REG0;\
>> -uint32_t r1 = adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG1;\
>> -uint32_t spare_int = adev-
>>> reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] +
>> mmRLC_SPARE_INT;\
>> -WREG32(r0, value);\
>> -WREG32(r1, (reg | 0x80000000));\
>> -WREG32(spare_int, 0x1);\
>> -for (i = 0; i < retries; i++) {\
>> -u32 tmp = RREG32(r1);\
>> -if (!(tmp & 0x80000000))\
>> -break;\
>> -udelay(10);\
>> -}\
>> -if (i >= retries)\
>> -pr_err("timeout: rlcg program reg:0x%05x failed !\n", reg);\ -} else
>> {\ -WREG32(reg, value); \ -}\
>> +do { \
>> +if (adev->gfx.rlc.funcs->rlcg_wreg) \
>> +adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value, 0); \
>> +else \
>> +WREG32(reg, value);\
>> } while (0)
>>
>> #define WREG32_RLC_EX(prefix, reg, value) \ @@ -125,23 +109,24 @@ }
>> while (0)
>>
>> #define WREG32_SOC15_RLC_SHADOW(ip, inst, reg, value) \ -do {\
>> -uint32_t target_reg = adev-
>>> reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
>> -if (amdgpu_sriov_fullaccess(adev)) {    \
>> -uint32_t r2 = adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG2;\
>> -uint32_t r3 = adev-
>>> reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] +
>> mmSCRATCH_REG3;\
>> -uint32_t grbm_cntl = adev-
>>> reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] +
>> mmGRBM_GFX_CNTL;   \
>> -uint32_t grbm_idx = adev-
>>> reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] +
>> mmGRBM_GFX_INDEX;   \
>> -if (target_reg == grbm_cntl) \
>> -WREG32(r2, value);\
>> -else if (target_reg == grbm_idx) \
>> -WREG32(r3, value);\
>> -WREG32(target_reg, value);\
>> -} else {\
>> -WREG32(target_reg, value); \
>> -}\
>> +WREG32_RLC((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>> reg),
>> +value)
>> +
>> +#define RREG32_RLC(reg) \
>> +(adev->gfx.rlc.funcs->rlcg_rreg ? \
>> +adev->gfx.rlc.funcs->rlcg_rreg(adev, reg, 0) : RREG32(reg))
>> +
>> +#define WREG32_RLC_NO_KIQ(reg, value) \ do { \ if
>> +(adev->gfx.rlc.funcs->rlcg_wreg) \
>> +adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, value,
>> AMDGPU_REGS_NO_KIQ); \
>> +else \
>> +WREG32_NO_KIQ(reg, value);\
>> } while (0)
>>
>> +#define RREG32_RLC_NO_KIQ(reg) \
>> +(adev->gfx.rlc.funcs->rlcg_rreg ? \
>> +adev->gfx.rlc.funcs->rlcg_rreg(adev, reg,
>> AMDGPU_REGS_NO_KIQ) :
>> +RREG32_NO_KIQ(reg))
>> +
>> #define WREG32_SOC15_RLC_SHADOW_EX(prefix, ip, inst, reg, value) \ do
>> {\ uint32_t target_reg = adev-
>>> reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\ @@ -160,10 +145,13
>> @@
>> }\
>> } while (0)
>>
>> +#define RREG32_SOC15_RLC(ip, inst, reg) \
>> +RREG32_RLC(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>> reg)
>> +
>> #define WREG32_SOC15_RLC(ip, inst, reg, value) \  do {\ -uint32_t
>> target_reg = adev-
>>> reg_offset[GC_HWIP][0][reg##_BASE_IDX] + reg;\
>> -WREG32_RLC(target_reg, value); \
>> +uint32_t target_reg = adev-
>>> reg_offset[ip##_HWIP][0][reg##_BASE_IDX] + reg;\
>> +WREG32_RLC(target_reg, value); \
>> } while (0)
>>
>> #define WREG32_SOC15_RLC_EX(prefix, ip, inst, reg, value) \ @@ -173,11
>> +161,14 @@
>> } while (0)
>>
>> #define WREG32_FIELD15_RLC(ip, idx, reg, field, val)   \
>> -    WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>> mm##reg), \
>> -    (RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] +
>> mm##reg) \
>> -    & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>> +WREG32_RLC((adev-
>>> reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
>> +(RREG32_RLC(adev-
>>> reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
>> +& ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
>>
>> #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
>> -    WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
>> + offset), value)
>> +WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>> reg)
>> ++ offset), value)
>> +
>> +#define RREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset) \
>> +RREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] +
>> reg)
>> ++ offset))
>>
>> #endif
>> --
>> 2.17.1
>>
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists
>> .fr
>> eedesktop.org/mailman/listinfo/amd-
>> gfx
>> b1c08d8f404c823%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C63
>> 7527648681476268%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAi
>> LCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=SIC
>> r%2FbhwrejJzpuhJmsnJiu6TP%2Fvu075po7%2BJ70Foec%3D&amp;reserved=0
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx