[V3 00/11] Add support AMD GPU virtualization soultion

[V3 00/11] Add support AMD GPU virtualization soultion

[Xiangliang Yu]

This patch series will enable AMD GPU virtualization and wires it
into amdgpu modules.

Changes in V3:
1. rename field name;
2. remove extra fence_get()/fence_put();
3. fixed coding style;
4. update patch comment;

Changes in V2:
1. define new high level interface for non-asic specific;
2. add asic specific file to support specific asic;
3. not include KIQ patch as it has been merged into kernel;
4. not include CSA patch as monk will submit it;

Xiangliang Yu (11):
  drm/amdgpu/ring: add two interfaces to support r/w registers with kiq
  drm/amdgpu/gfx8: implement emit_rreg/wreg function
  drm/amdgpu/virt: add runtime flag
  drm/amdgpu/virt: use kiq to access registers
  drm/amdgpu/virt: add high level interfaces for virt
  drm/amdgpu/virt: implement VI virt operation interfaces
  drm/amdgpu: request/release full gpu access if device is vf
  drm/amdgpu/vi: add support virtualization
  drm/amdgpu/virt: enable virtual display
  drm/amdgpu/virt: disable dc
  drm/amdgpu: do not reset gpu for virtualization

 drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592 +++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
 11 files changed, 926 insertions(+), 4 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 01/11] drm/amdgpu/ring: add two interfaces to support r/w registers with kiq

[Xiangliang Yu]

During virtual runtime, need to send command to kiq ring to
read/write GPU registers. Add two interface to support the two
actions.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Linu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h      | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 13c30ab..13d87d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1646,6 +1646,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
 #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
+#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
+#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 92bc89b..c813cbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -135,6 +135,8 @@ struct amdgpu_ring_funcs {
 	void (*end_use)(struct amdgpu_ring *ring);
 	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
 	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
+	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 };
 
 struct amdgpu_ring {
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg function

[Xiangliang Yu]

Implement emit_rreg/wreg function for kiq ring.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  3 ++-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 0d821d9..003bffb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -30,7 +30,8 @@
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t caps;
+	uint32_t		caps;
+	uint32_t		reg_val_offs;
 };
 
 #define amdgpu_sriov_enabled(adev) \
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index b487b97..a18b392 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1373,6 +1373,12 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 {
 	int r = 0;
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
+		if (r)
+			return r;
+	}
+
 	ring->adev = NULL;
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
@@ -1399,6 +1405,9 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
 				   struct amdgpu_irq_src *irq)
 {
+	if (amdgpu_sriov_vf(ring->adev))
+		amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
+
 	amdgpu_ring_fini(ring);
 	irq->data = NULL;
 }
@@ -6681,6 +6690,32 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 	amdgpu_ring_write(ring, 0);
 }
 
+static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+	amdgpu_ring_write(ring, 0 |	/* src: register*/
+				(5 << 8) |	/* dst: memory */
+				(1 << 20));	/* write confirm */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+				adev->virt.reg_val_offs * 4));
+	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+				adev->virt.reg_val_offs * 4));
+}
+
+static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+				  uint32_t val)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, val);
+}
+
 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 						 enum amdgpu_interrupt_state state)
 {
@@ -6995,6 +7030,8 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_rreg = gfx_v8_0_ring_emit_rreg,
+	.emit_wreg = gfx_v8_0_ring_emit_wreg,
 };
 
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 03/11] drm/amdgpu/virt: add runtime flag

[Xiangliang Yu]

Add new flag to define gpu runtime that is out of full gpu access.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 003bffb..3d38a9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -28,6 +28,7 @@
 #define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled on this GPU */
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
+#define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
@@ -43,6 +44,9 @@ struct amdgpu_virt {
 #define amdgpu_sriov_bios(adev) \
 ((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
 
+#define amdgpu_sriov_runtime(adev) \
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME)
+
 #define amdgpu_passthrough(adev) \
 ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Xiangliang Yu]

For virtualization, it is must for driver to use KIQ to access
registers when it is out of GPU full access mode.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
 5 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 4185b03..0b8e470 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 {
 	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 {
 	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
new file mode 100644
index 0000000..7861b6b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev)
+{
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
+{
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 3d38a9f..2869980 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -33,6 +33,7 @@
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
+	struct mutex		lock;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
 #endif
 }
 
+void amdgpu_virt_init_setting(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 7350a8f..dc0d4fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
 	switch (adev->asic_type) {
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Xiangliang Yu]

Add high level interfaces that is not relate to specific asic. So
asic files just need to implement the interfaces to support
virtualization.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 7861b6b..4e7df8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 2869980..45771b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations
+ */
+struct amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev);
+};
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Xiangliang Yu]

VI has asic specific virt support, which including mailbox and
golden registers init.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 0b8e470..388e7b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 45771b0..a409eec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
new file mode 100644
index 0000000..d2622b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Xiangliang.Yu@amd.com
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
+};
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+};
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_settings_a10,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_settings_a10));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_common_all));
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_settings_a11,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_settings_a11));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_common_all));
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs
+ */
+static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool val)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true);
+}
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev)
+{
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event event)
+{
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev)
+{
+	return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
+}
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
+	return xgpu_vi_send_access_requests(adev, event);
+}
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state)
+{
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
+{
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */
+}
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state)
+{
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs;
+}
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work, xgpu_vi_mailbox_flr_work);
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev)
+{
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+}
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
new file mode 100644
index 0000000..fd6216e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 07/11] drm/amdgpu: request/release full gpu access if device is vf

[Xiangliang Yu]

For gpu vf device, first need to request full gpu access before
accessing gpu registers, and release full gpu access after the
access is done.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 9 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 9a2fd3e..03d28a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1330,6 +1330,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_request_full_gpu(adev, true);
+		if (r)
+			return r;
+	}
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
 			DRM_ERROR("disabled ip block: %d\n", i);
@@ -1523,6 +1529,9 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.late_initialized = false;
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, false);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 47bc8e1..eb228c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -60,6 +60,9 @@ int amdgpu_driver_unload_kms(struct drm_device *dev)
 	if (adev->rmmio == NULL)
 		goto done_free;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
 	if (amdgpu_device_is_px(dev)) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
@@ -139,6 +142,9 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 		pm_runtime_put_autosuspend(dev->dev);
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, true);
+
 out:
 	if (r) {
 		/* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 08/11] drm/amdgpu/vi: add support virtualization

[Xiangliang Yu]

Call VI virtualization functions if device is Vf.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/vi.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index dc0d4fa..34aa603 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -77,6 +77,7 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_powerplay.h"
 #include "dce_virtual.h"
+#include "mxgpu_vi.h"
 
 /*
  * Indirect registers accessor
@@ -273,6 +274,12 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 	mutex_lock(&adev->grbm_idx_mutex);
 
+	if (amdgpu_sriov_vf(adev)) {
+		xgpu_vi_init_golden_registers(adev);
+		mutex_unlock(&adev->grbm_idx_mutex);
+		return;
+	}
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		amdgpu_program_register_sequence(adev,
@@ -892,8 +899,10 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
-	if (amdgpu_sriov_vf(adev))
+	if (amdgpu_sriov_vf(adev)) {
 		amdgpu_virt_init_setting(adev);
+		xgpu_vi_mailbox_set_irq_funcs(adev);
+	}
 
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
@@ -1059,8 +1068,23 @@ static int vi_common_early_init(void *handle)
 	return 0;
 }
 
+static int vi_common_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_get_irq(adev);
+
+	return 0;
+}
+
 static int vi_common_sw_init(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_add_irq_id(adev);
+
 	return 0;
 }
 
@@ -1092,6 +1116,9 @@ static int vi_common_hw_fini(void *handle)
 	/* enable the doorbell aperture */
 	vi_enable_doorbell_aperture(adev, false);
 
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_put_irq(adev);
+
 	return 0;
 }
 
@@ -1376,7 +1403,7 @@ static int vi_common_set_powergating_state(void *handle,
 static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.name = "vi_common",
 	.early_init = vi_common_early_init,
-	.late_init = NULL,
+	.late_init = vi_common_late_init,
 	.sw_init = vi_common_sw_init,
 	.sw_fini = vi_common_sw_fini,
 	.hw_init = vi_common_hw_init,
@@ -1404,6 +1431,9 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 	/* in early init stage, vbios code won't work */
 	vi_detect_hw_virtualization(adev);
 
+	if (amdgpu_sriov_vf(adev))
+		adev->virt.ops = &xgpu_vi_virt_ops;
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		/* topaz has no DCE, UVD, VCE */
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 09/11] drm/amdgpu/virt: enable virtual display

[Xiangliang Yu]

Virtual display is default setting for virtualization, enable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 4e7df8e..4bc373e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -25,6 +25,10 @@
 
 void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 {
+	/* enable virtual display */
+	adev->mode_info.num_crtc = 1;
+	adev->enable_virtual_display = true;
+
 	mutex_init(&adev->virt.lock);
 }
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 10/11] drm/amdgpu/virt: disable dc

[Xiangliang Yu]

Virtualization don't need the dc, disable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 03d28a1..a6d828c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
  */
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
 }
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V3 11/11] drm/amdgpu: do not reset gpu for virtualization

[Xiangliang Yu]

Current amdgpu reset process only works on bare-metal and for
SRIOV many inside it need re-work to adapt to vf device.

This is a temporary workaround to skip gpu reset.

Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index a6d828c..5705fbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 	int resched;
 	bool need_full_reset;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	if (!amdgpu_check_soft_reset(adev)) {
 		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
 		return 0;
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 97 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>   	uint32_t ret;
>   
> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +		return amdgpu_virt_kiq_rreg(adev, reg);
> +

Mhm, why is the in_interrupt() necessary here?

>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>   
> +	if (amdgpu_sriov_runtime(adev))
> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> +

And why it isn't necessary here? What happens when we write a register 
from an interrupt routine?

Christian.

>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..7861b6b
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,82 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev)
> +{
> +	mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
> +{
> +	signed long r;
> +	uint32_t val;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_rreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +
> +	val = adev->wb.wb[adev->virt.reg_val_offs];
> +
> +	return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
> +{
> +	signed long r;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_wreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_wreg(ring, reg, v);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 3d38a9f..2869980 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		reg_val_offs;
> +	struct mutex		lock;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \
> @@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
>   #endif
>   }
>   
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev);
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
> index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>   		smc_enabled = true;
>   
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_setting(adev);
> +
>   	adev->rev_id = vi_get_rev_id(adev);
>   	adev->external_rev_id = 0xFF;
>   	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V3 00/11] Add support AMD GPU virtualization soultion

[Christian König]

Patches #1-#3, #5 and #8 are Reviewed-by: Christian König 
<christian.koenig@amd.com>.

Patch #6, #9 and #10 are Acked-by: Christian König 
<christian.koenig@amd.com>.

Patch #4: Just send out a question on this.

Patch #7 is missing a comment that this applies during driver 
load/unload, cause that isn't obvious.

With that fixed the patch is Reviewed-by: Christian König 
<christian.koenig@amd.com> as well.

Patch #10 please add the comment that this is just a temporary 
workaround to the code as well.

With that fixed the patch is also Reviewed-by: Christian König 
<christian.koenig@amd.com>.

Regards,
Christian.

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
> This patch series will enable AMD GPU virtualization and wires it
> into amdgpu modules.
>
> Changes in V3:
> 1. rename field name;
> 2. remove extra fence_get()/fence_put();
> 3. fixed coding style;
> 4. update patch comment;
>
> Changes in V2:
> 1. define new high level interface for non-asic specific;
> 2. add asic specific file to support specific asic;
> 3. not include KIQ patch as it has been merged into kernel;
> 4. not include CSA patch as monk will submit it;
>
> Xiangliang Yu (11):
>    drm/amdgpu/ring: add two interfaces to support r/w registers with kiq
>    drm/amdgpu/gfx8: implement emit_rreg/wreg function
>    drm/amdgpu/virt: add runtime flag
>    drm/amdgpu/virt: use kiq to access registers
>    drm/amdgpu/virt: add high level interfaces for virt
>    drm/amdgpu/virt: implement VI virt operation interfaces
>    drm/amdgpu: request/release full gpu access if device is vf
>    drm/amdgpu/vi: add support virtualization
>    drm/amdgpu/virt: enable virtual display
>    drm/amdgpu/virt: disable dc
>    drm/amdgpu: do not reset gpu for virtualization
>
>   drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
>   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592 +++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
>   11 files changed, 926 insertions(+), 4 deletions(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )
 
BR Monk

-----Original Message-----
From: Christian König [mailto:deathsimple@vodafone.de] 
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Liu, Monk <Monk.Liu@amd.com>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access 
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 97 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>   	uint32_t ret;
>   
> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +		return amdgpu_virt_kiq_rreg(adev, reg);
> +

Mhm, why is the in_interrupt() necessary here?

>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>   
> +	if (amdgpu_sriov_runtime(adev))
> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> +

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..7861b6b
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,82 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person 
> +obtaining a
> + * copy of this software and associated documentation files (the 
> +"Software"),
> + * to deal in the Software without restriction, including without 
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute, 
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom 
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be 
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> +	mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t 
> +reg) {
> +	signed long r;
> +	uint32_t val;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_rreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +
> +	val = adev->wb.wb[adev->virt.reg_val_offs];
> +
> +	return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v) {
> +	signed long r;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_wreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_wreg(ring, reg, v);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 3d38a9f..2869980 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		reg_val_offs;
> +	struct mutex		lock;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline 
> bool is_virtual_machine(void)
>   #endif
>   }
>   
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c 
> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>   		smc_enabled = true;
>   
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_setting(adev);
> +
>   	adev->rev_id = vi_get_rev_id(adev);
>   	adev->external_rev_id = 0xFF;
>   	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 00/11] Add support AMD GPU virtualization soultion

[Yu, Xiangliang]

> Patch #7 is missing a comment that this applies during driver load/unload,
> cause that isn't obvious.

Not just only for loading/unloading, resume process will need to request/release full gpu access after FLR (function level reset). Later I'll upstream FLR patches.

> 
> With that fixed the patch is Reviewed-by: Christian König
> <christian.koenig@amd.com> as well.
> 
> Patch #10 please add the comment that this is just a temporary workaround
> to the code as well.
> 
> With that fixed the patch is also Reviewed-by: Christian König
> <christian.koenig@amd.com>.
> 
> Regards,
> Christian.
> 
> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
> > This patch series will enable AMD GPU virtualization and wires it into
> > amdgpu modules.
> >
> > Changes in V3:
> > 1. rename field name;
> > 2. remove extra fence_get()/fence_put(); 3. fixed coding style; 4.
> > update patch comment;
> >
> > Changes in V2:
> > 1. define new high level interface for non-asic specific; 2. add asic
> > specific file to support specific asic; 3. not include KIQ patch as it
> > has been merged into kernel; 4. not include CSA patch as monk will
> > submit it;
> >
> > Xiangliang Yu (11):
> >    drm/amdgpu/ring: add two interfaces to support r/w registers with kiq
> >    drm/amdgpu/gfx8: implement emit_rreg/wreg function
> >    drm/amdgpu/virt: add runtime flag
> >    drm/amdgpu/virt: use kiq to access registers
> >    drm/amdgpu/virt: add high level interfaces for virt
> >    drm/amdgpu/virt: implement VI virt operation interfaces
> >    drm/amdgpu: request/release full gpu access if device is vf
> >    drm/amdgpu/vi: add support virtualization
> >    drm/amdgpu/virt: enable virtual display
> >    drm/amdgpu/virt: disable dc
> >    drm/amdgpu: do not reset gpu for virtualization
> >
> >   drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
> >   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
> >   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592
> +++++++++++++++++++++++++++++
> >   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
> >   drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
> >   11 files changed, 926 insertions(+), 4 deletions(-)
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Liu, Monk]

Hi Xiangliang

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, 
Seems you add an upper layer interface to invoke on gpu_request/release routines,
And I will also take role in the reviewing of it:

+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return function)


+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME"  should be put to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)
    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check hypervisor please */
return r;

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com] 
Sent: Wednesday, January 11, 2017 9:18 PM
To: amd-gfx@lists.freedesktop.org
Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk <Monk.Liu@amd.com>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 7861b6b..4e7df8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 2869980..45771b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
That won't work. Locking a mutext like it is done in the write path can 
cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need 
to change the code to use a spinlock with disabled interrupts instead of 
a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens 
to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a 
bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:
> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>
> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>
> And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )
>   
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 9:38 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Liu, Monk <Monk.Liu@amd.com>
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>> For virtualization, it is must for driver to use KIQ to access
>> registers when it is out of GPU full access mode.
>>
>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>> ---
>>    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>    5 files changed, 97 insertions(+), 1 deletion(-)
>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>> index 4185b03..0b8e470 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>    
>>    # add asic specific block
>>    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index f82919d..9a2fd3e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>    {
>>    	uint32_t ret;
>>    
>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>> +		return amdgpu_virt_kiq_rreg(adev, reg);
>> +
> Mhm, why is the in_interrupt() necessary here?
>
>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>    	else {
>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>    {
>>    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>    
>> +	if (amdgpu_sriov_runtime(adev))
>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
>> +
> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>
> Christian.
>
>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>    	else {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> new file mode 100644
>> index 0000000..7861b6b
>> --- /dev/null
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> @@ -0,0 +1,82 @@
>> +/*
>> + * Copyright 2017 Advanced Micro Devices, Inc.
>> + *
>> + * Permission is hereby granted, free of charge, to any person
>> +obtaining a
>> + * copy of this software and associated documentation files (the
>> +"Software"),
>> + * to deal in the Software without restriction, including without
>> +limitation
>> + * the rights to use, copy, modify, merge, publish, distribute,
>> +sublicense,
>> + * and/or sell copies of the Software, and to permit persons to whom
>> +the
>> + * Software is furnished to do so, subject to the following conditions:
>> + *
>> + * The above copyright notice and this permission notice shall be
>> +included in
>> + * all copies or substantial portions of the Software.
>> + *
>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>> +EXPRESS OR
>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>> +MERCHANTABILITY,
>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT
>> +SHALL
>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>> +DAMAGES OR
>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>> +OTHERWISE,
>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
>> +OR
>> + * OTHER DEALINGS IN THE SOFTWARE.
>> + */
>> +
>> +#include "amdgpu.h"
>> +#include "amdgpu_virt.h"
>> +
>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>> +	mutex_init(&adev->virt.lock);
>> +}
>> +
>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>> +reg) {
>> +	signed long r;
>> +	uint32_t val;
>> +	struct fence *f;
>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> +	struct amdgpu_ring *ring = &kiq->ring;
>> +
>> +	BUG_ON(!ring->funcs->emit_rreg);
>> +
>> +	mutex_lock(&adev->virt.lock);
>> +	amdgpu_ring_alloc(ring, 32);
>> +	amdgpu_ring_emit_hdp_flush(ring);
>> +	amdgpu_ring_emit_rreg(ring, reg);
>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>> +	amdgpu_fence_emit(ring, &f);
>> +	amdgpu_ring_commit(ring);
>> +	mutex_unlock(&adev->virt.lock);
>> +
>> +	r = fence_wait(f, false);
>> +	if (r)
>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>> +	fence_put(f);
>> +
>> +	val = adev->wb.wb[adev->virt.reg_val_offs];
>> +
>> +	return val;
>> +}
>> +
>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>> +uint32_t v) {
>> +	signed long r;
>> +	struct fence *f;
>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> +	struct amdgpu_ring *ring = &kiq->ring;
>> +
>> +	BUG_ON(!ring->funcs->emit_wreg);
>> +
>> +	mutex_lock(&adev->virt.lock);
>> +	amdgpu_ring_alloc(ring, 32);
>> +	amdgpu_ring_emit_hdp_flush(ring);
>> +	amdgpu_ring_emit_wreg(ring, reg, v);
>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>> +	amdgpu_fence_emit(ring, &f);
>> +	amdgpu_ring_commit(ring);
>> +	mutex_unlock(&adev->virt.lock);
>> +
>> +	r = fence_wait(f, false);
>> +	if (r)
>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>> +	fence_put(f);
>> +}
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> index 3d38a9f..2869980 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> @@ -33,6 +33,7 @@
>>    struct amdgpu_virt {
>>    	uint32_t		caps;
>>    	uint32_t		reg_val_offs;
>> +	struct mutex		lock;
>>    };
>>    
>>    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline
>> bool is_virtual_machine(void)
>>    #endif
>>    }
>>    
>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
>> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>> +uint32_t v);
>> +
>>    #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>    		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>    		smc_enabled = true;
>>    
>> +	if (amdgpu_sriov_vf(adev))
>> +		amdgpu_virt_init_setting(adev);
>> +
>>    	adev->rev_id = vi_get_rev_id(adev);
>>    	adev->external_rev_id = 0xFF;
>>    	switch (adev->asic_type) {
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Yu, Xiangliang]

> This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add
> my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer
> interface to invoke on gpu_request/release routines, And I will also take role
> in the reviewing of it:
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> 
> [ml]One nitpick , you need place " adev->virt.caps &=
> ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

> 
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> 
> [ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is designed not
> allowed to fail then you should type it as non-return function. ( I remember
> in my first initial implementation on rel_gpu function it is is non-return
> function)

I think it is more flexible, we can handle error according to return result.

> 
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> 
> [ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME"  should be
> put to the second line as well, like:
> 
> r = virt-ops->reset_gpu(adev);
> If (!r)
>     adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
>     BUG(); /* we shouldn't fail in requesting GPU_RESET, check hypervisor
> please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

> 
> BR Monk
> 
> -----Original Message-----
> From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> Sent: Wednesday, January 11, 2017 9:18 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk
> <Monk.Liu@amd.com>
> Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> Add high level interfaces that is not relate to specific asic. So asic files just
> need to implement the interfaces to support virtualization.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> ++++++++++++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
>  2 files changed, 71 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 7861b6b..4e7df8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
> *adev, uint32_t reg, uint32_t v)
>  		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>  	fence_put(f);
>  }
> +
> +/**
> + * amdgpu_virt_request_full_gpu() - request full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When start to init/fini driver, first need to request full gpu access.
> + * Return: Zero if request success, otherwise will return error.
> + */
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->req_full_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_release_full_gpu() - release full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When finishing driver init/fini, need to release full gpu access.
> + * Return: Zero if release success, otherwise will returen error.
> + */
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +	int r;
> +
> +	if (virt->ops && virt->ops->rel_full_gpu) {
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return r;
> +	}
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_reset_gpu() - reset gpu
> + * @amdgpu:	amdgpu device.
> + * Send reset command to GPU hypervisor to reset GPU that VM is using
> + * Return: Zero if reset success, otherwise will return error.
> + */
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->reset_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> +
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 2869980..45771b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -29,11 +29,22 @@
>  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
>  #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access
> mode */
> +
> +/**
> + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
> +amdgpu_virt_ops {
> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*reset_gpu)(struct amdgpu_device *adev); };
> +
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t		caps;
>  	uint32_t		reg_val_offs;
>  	struct mutex		lock;
> +	const struct amdgpu_virt_ops	*ops;
>  };
> 
>  #define amdgpu_sriov_enabled(adev) \
> @@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void
> amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t
> amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void
> amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t
> v);
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
> +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> 
>  #endif
> --
> 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Liu, Monk]

+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is IDH_FLR_NOTIFICATION_CMPL, 

The previous one means GIM told us GIM will start do FLR
The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL, 

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com] 
Sent: Wednesday, January 11, 2017 9:18 PM
To: amd-gfx@lists.freedesktop.org
Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 0b8e470..388e7b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 45771b0..a409eec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
new file mode 100644
index 0000000..d2622b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Xiangliang.Yu@amd.com
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_settings_a10,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_settings_a10));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_common_all));
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_settings_a11,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_settings_a11));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_common_all));
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ static void 
+xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool 
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event 
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS); }
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work, xgpu_vi_mailbox_flr_work);
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
new file mode 100644
index 0000000..fd6216e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void 
+xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int 
+xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int 
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void 
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Liu, Monk]

See in lines


-----Original Message-----
From: Yu, Xiangliang 
Sent: Wednesday, January 11, 2017 10:27 PM
To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

> This patch is not derived from my work (amd-sriov-4.3/4.6), please 
> don't add my "signed-off-by: Monk Liu " line on it, Seems you add an 
> upper layer interface to invoke on gpu_request/release routines, And I 
> will also take role in the reviewing of it:
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> 
> [ml]One nitpick , you need place " adev->virt.caps &= 
> ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

[ml] NO, you didn't get my point, you should &= ~ AMDGPU_SRIOV_CAPS_RUNTIME *after* virt->ops->req_full_gpu(adev, init)

> 
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> 
> [ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is 
> designed not allowed to fail then you should type it as non-return 
> function. ( I remember in my first initial implementation on rel_gpu 
> function it is is non-return
> function)

I think it is more flexible, we can handle error according to return result.

[ML] then please handle the error accordingly, but your patch just ignore the return value . see it yourself 

> 
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> 
> [ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME"  should be put 
> to the second line as well, like:
> 
> r = virt-ops->reset_gpu(adev);
> If (!r)
>     adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
>     BUG(); /* we shouldn't fail in requesting GPU_RESET, check 
> hypervisor please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

[ML] you still not get my point, if reset_gpu() failed. You shouldn't "&= ~AMDGPU_SRIOV_CAPS_RUNTIME", but you did it in prior to reset_gpu(), isn't it ??

> 
> BR Monk
> 
> -----Original Message-----
> From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> Sent: Wednesday, January 11, 2017 9:18 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk 
> <Monk.Liu@amd.com>
> Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for 
> virt
> 
> Add high level interfaces that is not relate to specific asic. So asic 
> files just need to implement the interfaces to support virtualization.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> ++++++++++++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
>  2 files changed, 71 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 7861b6b..4e7df8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device 
> *adev, uint32_t reg, uint32_t v)
>  		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>  	fence_put(f);
>  }
> +
> +/**
> + * amdgpu_virt_request_full_gpu() - request full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When start to init/fini driver, first need to request full gpu access.
> + * Return: Zero if request success, otherwise will return error.
> + */
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
> +init) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->req_full_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_release_full_gpu() - release full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When finishing driver init/fini, need to release full gpu access.
> + * Return: Zero if release success, otherwise will returen error.
> + */
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool 
> +init) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +	int r;
> +
> +	if (virt->ops && virt->ops->rel_full_gpu) {
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return r;
> +	}
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_reset_gpu() - reset gpu
> + * @amdgpu:	amdgpu device.
> + * Send reset command to GPU hypervisor to reset GPU that VM is using
> + * Return: Zero if reset success, otherwise will return error.
> + */
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->reset_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> +
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 2869980..45771b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -29,11 +29,22 @@
>  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
>  #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access
> mode */
> +
> +/**
> + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
> +amdgpu_virt_ops {
> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*reset_gpu)(struct amdgpu_device *adev); };
> +
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t		caps;
>  	uint32_t		reg_val_offs;
>  	struct mutex		lock;
> +	const struct amdgpu_virt_ops	*ops;
>  };
> 
>  #define amdgpu_sriov_enabled(adev) \
> @@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void 
> amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t 
> amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void 
> amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> uint32_t v);
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
> +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
> +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> 
>  #endif
> --
> 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

Yeah, make sense 

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:deathsimple@vodafone.de] 
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:
> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>
> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>
> And to fully support register reading, we need to implement a new 
> method to use KIQ access register without any schedule() chance, but 
> that's overhead for current stage, and we don't observe registers 
> reading in IRQ context now on 3D apps. (kfd observed such case, and 
> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register 
> dumping, instead of directly reading them via KIQ )
>   
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 9:38 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; 
> amd-gfx@lists.freedesktop.org
> Cc: Liu, Monk <Monk.Liu@amd.com>
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>> For virtualization, it is must for driver to use KIQ to access 
>> registers when it is out of GPU full access mode.
>>
>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>> ---
>>    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>    5 files changed, 97 insertions(+), 1 deletion(-)
>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>> index 4185b03..0b8e470 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>    
>>    # add asic specific block
>>    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o 
>> \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index f82919d..9a2fd3e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>    {
>>    	uint32_t ret;
>>    
>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>> +		return amdgpu_virt_kiq_rreg(adev, reg);
>> +
> Mhm, why is the in_interrupt() necessary here?
>
>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>    	else {
>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>    {
>>    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>    
>> +	if (amdgpu_sriov_runtime(adev))
>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
>> +
> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>
> Christian.
>
>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>    	else {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> new file mode 100644
>> index 0000000..7861b6b
>> --- /dev/null
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> @@ -0,0 +1,82 @@
>> +/*
>> + * Copyright 2017 Advanced Micro Devices, Inc.
>> + *
>> + * Permission is hereby granted, free of charge, to any person 
>> +obtaining a
>> + * copy of this software and associated documentation files (the 
>> +"Software"),
>> + * to deal in the Software without restriction, including without 
>> +limitation
>> + * the rights to use, copy, modify, merge, publish, distribute, 
>> +sublicense,
>> + * and/or sell copies of the Software, and to permit persons to whom 
>> +the
>> + * Software is furnished to do so, subject to the following conditions:
>> + *
>> + * The above copyright notice and this permission notice shall be 
>> +included in
>> + * all copies or substantial portions of the Software.
>> + *
>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
>> +EXPRESS OR
>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
>> +MERCHANTABILITY,
>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO 
>> +EVENT SHALL
>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
>> +DAMAGES OR
>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
>> +OTHERWISE,
>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
>> +USE OR
>> + * OTHER DEALINGS IN THE SOFTWARE.
>> + */
>> +
>> +#include "amdgpu.h"
>> +#include "amdgpu_virt.h"
>> +
>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>> +	mutex_init(&adev->virt.lock);
>> +}
>> +
>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>> +reg) {
>> +	signed long r;
>> +	uint32_t val;
>> +	struct fence *f;
>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> +	struct amdgpu_ring *ring = &kiq->ring;
>> +
>> +	BUG_ON(!ring->funcs->emit_rreg);
>> +
>> +	mutex_lock(&adev->virt.lock);
>> +	amdgpu_ring_alloc(ring, 32);
>> +	amdgpu_ring_emit_hdp_flush(ring);
>> +	amdgpu_ring_emit_rreg(ring, reg);
>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>> +	amdgpu_fence_emit(ring, &f);
>> +	amdgpu_ring_commit(ring);
>> +	mutex_unlock(&adev->virt.lock);
>> +
>> +	r = fence_wait(f, false);
>> +	if (r)
>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>> +	fence_put(f);
>> +
>> +	val = adev->wb.wb[adev->virt.reg_val_offs];
>> +
>> +	return val;
>> +}
>> +
>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
>> +uint32_t v) {
>> +	signed long r;
>> +	struct fence *f;
>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>> +	struct amdgpu_ring *ring = &kiq->ring;
>> +
>> +	BUG_ON(!ring->funcs->emit_wreg);
>> +
>> +	mutex_lock(&adev->virt.lock);
>> +	amdgpu_ring_alloc(ring, 32);
>> +	amdgpu_ring_emit_hdp_flush(ring);
>> +	amdgpu_ring_emit_wreg(ring, reg, v);
>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>> +	amdgpu_fence_emit(ring, &f);
>> +	amdgpu_ring_commit(ring);
>> +	mutex_unlock(&adev->virt.lock);
>> +
>> +	r = fence_wait(f, false);
>> +	if (r)
>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>> +	fence_put(f);
>> +}
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> index 3d38a9f..2869980 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>> @@ -33,6 +33,7 @@
>>    struct amdgpu_virt {
>>    	uint32_t		caps;
>>    	uint32_t		reg_val_offs;
>> +	struct mutex		lock;
>>    };
>>    
>>    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static 
>> inline bool is_virtual_machine(void)
>>    #endif
>>    }
>>    
>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
>> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
>> +uint32_t v);
>> +
>>    #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c 
>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>    		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>    		smc_enabled = true;
>>    
>> +	if (amdgpu_sriov_vf(adev))
>> +		amdgpu_virt_init_setting(adev);
>> +
>>    	adev->rev_id = vi_get_rev_id(adev);
>>    	adev->external_rev_id = 0xFF;
>>    	switch (adev->asic_type) {
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Liu, Monk]

> 
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> 
> [ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is 
> designed not allowed to fail then you should type it as non-return 
> function. ( I remember in my first initial implementation on rel_gpu 
> function it is is non-return
> function)

I think it is more flexible, we can handle error according to return result.

[ML] then please handle the error accordingly, but you always do "|= AMDGPU_SRIOV_CAPS_RUNTIME" no matter above rel_full_gpu() return okay or not , isn't it  ?

That RUNTIME flag should only be set after successfully release the full access mode on GPU...


-----Original Message-----
From: Yu, Xiangliang 
Sent: Wednesday, January 11, 2017 10:27 PM
To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

> This patch is not derived from my work (amd-sriov-4.3/4.6), please 
> don't add my "signed-off-by: Monk Liu " line on it, Seems you add an 
> upper layer interface to invoke on gpu_request/release routines, And I 
> will also take role in the reviewing of it:
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> 
> [ml]One nitpick , you need place " adev->virt.caps &= 
> ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

> 
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> 
> [ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is 
> designed not allowed to fail then you should type it as non-return 
> function. ( I remember in my first initial implementation on rel_gpu 
> function it is is non-return
> function)

I think it is more flexible, we can handle error according to return result.

> 
> 
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> 
> [ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME"  should be put 
> to the second line as well, like:
> 
> r = virt-ops->reset_gpu(adev);
> If (!r)
>     adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
>     BUG(); /* we shouldn't fail in requesting GPU_RESET, check 
> hypervisor please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

> 
> BR Monk
> 
> -----Original Message-----
> From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> Sent: Wednesday, January 11, 2017 9:18 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk 
> <Monk.Liu@amd.com>
> Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for 
> virt
> 
> Add high level interfaces that is not relate to specific asic. So asic 
> files just need to implement the interfaces to support virtualization.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> ++++++++++++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
>  2 files changed, 71 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 7861b6b..4e7df8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device 
> *adev, uint32_t reg, uint32_t v)
>  		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>  	fence_put(f);
>  }
> +
> +/**
> + * amdgpu_virt_request_full_gpu() - request full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When start to init/fini driver, first need to request full gpu access.
> + * Return: Zero if request success, otherwise will return error.
> + */
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
> +init) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->req_full_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_release_full_gpu() - release full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When finishing driver init/fini, need to release full gpu access.
> + * Return: Zero if release success, otherwise will returen error.
> + */
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool 
> +init) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +	int r;
> +
> +	if (virt->ops && virt->ops->rel_full_gpu) {
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return r;
> +	}
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_reset_gpu() - reset gpu
> + * @amdgpu:	amdgpu device.
> + * Send reset command to GPU hypervisor to reset GPU that VM is using
> + * Return: Zero if reset success, otherwise will return error.
> + */
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->reset_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> +
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 2869980..45771b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -29,11 +29,22 @@
>  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
>  #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access
> mode */
> +
> +/**
> + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
> +amdgpu_virt_ops {
> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*reset_gpu)(struct amdgpu_device *adev); };
> +
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t		caps;
>  	uint32_t		reg_val_offs;
>  	struct mutex		lock;
> +	const struct amdgpu_virt_ops	*ops;
>  };
> 
>  #define amdgpu_sriov_enabled(adev) \
> @@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void 
> amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t 
> amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void 
> amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> uint32_t v);
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
> +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
> +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> 
>  #endif
> --
> 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> -----Original Message-----
> From: Liu, Monk
> Sent: Wednesday, January 11, 2017 10:28 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Shaoyun
> <Shaoyun.Liu@amd.com>
> Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> interfaces
> 
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> 
> [ML] we should be able to receive at least two type of IRQ from mailbox:
> One is FLR_NOTIFICATION and another one is
> IDH_FLR_NOTIFICATION_CMPL,
> 
> The previous one means GIM told us GIM will start do FLR The later one
> means GIM told us GIM already finished FLR and from that on Guest driver
> can do re-initialize .
> 
> You only handle the first IRQ, and dispatch a delay_work to monitor event of
> IDH_FLR_NOTIFICATION_CMPL,
> 
> I suspect that correct scheme may be you handle both of those two type of
> IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

> 
> BR Monk
> 
> -----Original Message-----
> From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> Sent: Wednesday, January 11, 2017 9:18 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk
> <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
> Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces
> 
> VI has asic specific virt support, which including mailbox and golden registers
> init.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> +++++++++++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>  4 files changed, 654 insertions(+), 4 deletions(-)  create mode 100644
> drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 0b8e470..388e7b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> cik_ih.o kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+=
> si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> 
>  amdgpu-y += \
> -	vi.o
> +	vi.o mxgpu_vi.o
> 
>  # add GMC block
>  amdgpu-y += \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 45771b0..a409eec 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
> 
>  /* GPU virtualization */
>  struct amdgpu_virt {
> -	uint32_t		caps;
> -	uint32_t		reg_val_offs;
> -	struct mutex		lock;
> +	uint32_t			caps;
> +	uint32_t			reg_val_offs;
> +	struct mutex			lock;
> +	struct amdgpu_irq_src		ack_irq;
> +	struct amdgpu_irq_src		rcv_irq;
> +	struct delayed_work		flr_work;
>  	const struct amdgpu_virt_ops	*ops;
>  };
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> new file mode 100644
> index 0000000..d2622b6
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> @@ -0,0 +1,592 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person
> +obtaining a
> + * copy of this software and associated documentation files (the
> +"Software"),
> + * to deal in the Software without restriction, including without
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute,
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors: Xiangliang.Yu@amd.com
> + */
> +
> +#include "amdgpu.h"
> +#include "vi.h"
> +#include "bif/bif_5_0_d.h"
> +#include "bif/bif_5_0_sh_mask.h"
> +#include "vid.h"
> +#include "gca/gfx_8_0_d.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "gmc_v8_0.h"
> +#include "gfx_v8_0.h"
> +#include "sdma_v3_0.h"
> +#include "tonga_ih.h"
> +#include "gmc/gmc_8_2_d.h"
> +#include "gmc/gmc_8_2_sh_mask.h"
> +#include "oss/oss_3_0_d.h"
> +#include "oss/oss_3_0_sh_mask.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "dce/dce_10_0_d.h"
> +#include "dce/dce_10_0_sh_mask.h"
> +#include "smu/smu_7_1_3_d.h"
> +#include "mxgpu_vi.h"
> +
> +/* VI golden setting */
> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> +
> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> +
> +static const u32 xgpu_fiji_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> +
> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> +
> +static const u32 xgpu_tonga_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> +	switch (adev->asic_type) {
> +	case CHIP_FIJI:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_settings_a10,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_settings_a10));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_common_all));
> +		break;
> +	case CHIP_TONGA:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_settings_a11,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_settings_a11));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_common_all));
> +		break;
> +	default:
> +		BUG_ON("Doesn't support chip type.\n");
> +		break;
> +	}
> +}
> +
> +/*
> + * Mailbox communication between GPU hypervisor and VFs  */ static void
> +xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool
> +val) {
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> +			    TRN_MSG_VALID, val ? 1 : 0);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> +				      enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> +			    MSGBUF_DATA, event);
> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> +
> +	xgpu_vi_mailbox_set_valid(adev, true); }
> +
> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> +				   enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> +	if (reg != event)
> +		return -ENOENT;
> +
> +	/* send ack to PF */
> +	xgpu_vi_mailbox_send_ack(adev);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	while (!(reg & mask)) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		reg = RREG32(mmMAILBOX_CONTROL);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event
> +event) {
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +
> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	while (r) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> +					enum idh_request request)
> +{
> +	int r;
> +
> +	xgpu_vi_mailbox_trans_msg(adev, request);
> +
> +	/* start to poll ack */
> +	r = xgpu_vi_poll_ack(adev);
> +	if (r)
> +		return r;
> +
> +	xgpu_vi_mailbox_set_valid(adev, false);
> +
> +	/* start to check msg if request is idh_req_gpu_init_access */
> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> +		if (r)
> +			return r;
> +	}
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> +	return xgpu_vi_send_access_requests(adev,
> IDH_REQ_GPU_RESET_ACCESS); }
> +
> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +
> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> IDH_REQ_GPU_FINI_ACCESS;
> +	return xgpu_vi_send_access_requests(adev, event); }
> +
> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +	int r = 0;
> +
> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> IDH_REL_GPU_FINI_ACCESS;
> +	r = xgpu_vi_send_access_requests(adev, event);
> +
> +	return r;
> +}
> +
> +/* add support mailbox interrupts */
> +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	DRM_DEBUG("get ack intr and do nothing.\n");
> +	return 0;
> +}
> +
> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state) {
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> +	struct amdgpu_virt *virt = container_of(work,
> +					struct amdgpu_virt, flr_work.work);
> +	struct amdgpu_device *adev = container_of(virt,
> +					struct amdgpu_device, virt);
> +	int r = 0;
> +
> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> +	if (r)
> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> +
> +	/* TODO: need to restore gfx states */ }
> +
> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state) {
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_ack_irq,
> +	.process = xgpu_vi_mailbox_ack_irq,
> +};
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> +	.process = xgpu_vi_mailbox_rcv_irq,
> +};
> +
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> +	adev->virt.ack_irq.num_types = 1;
> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> +	adev->virt.rcv_irq.num_types = 1;
> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> +
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> +	int r;
> +
> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> +	if (r)
> +		return r;
> +
> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	return 0;
> +}
> +
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> +	int r;
> +
> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> +	if (r)
> +		return r;
> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> xgpu_vi_mailbox_flr_work);
> +
> +	return 0;
> +}
> +
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> +
> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> +	.reset_gpu		= xgpu_vi_request_reset,
> +};
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> new file mode 100644
> index 0000000..fd6216e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> @@ -0,0 +1,55 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person
> +obtaining a
> + * copy of this software and associated documentation files (the
> +"Software"),
> + * to deal in the Software without restriction, including without
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute,
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef __MXGPU_VI_H__
> +#define __MXGPU_VI_H__
> +
> +#define VI_MAILBOX_TIMEDOUT	150
> +#define VI_MAILBOX_RESET_TIME	12
> +
> +/* VI mailbox messages request */
> +enum idh_request {
> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> +	IDH_REL_GPU_INIT_ACCESS,
> +	IDH_REQ_GPU_FINI_ACCESS,
> +	IDH_REL_GPU_FINI_ACCESS,
> +	IDH_REQ_GPU_RESET_ACCESS
> +};
> +
> +/* VI mailbox messages data */
> +enum idh_event {
> +	IDH_CLR_MSG_BUF = 0,
> +	IDH_READY_TO_ACCESS_GPU,
> +	IDH_FLR_NOTIFICATION,
> +	IDH_FLR_NOTIFICATION_CMPL,
> +	IDH_EVENT_MAX
> +};
> +
> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
> +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> +
> +#endif
> --
> 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

> do you think that's okay ?
Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup 
when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:
> Yeah, make sense
>
> I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?
>
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 10:16 PM
> To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
> That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.
>
> If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.
>
> Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.
>
> Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.
>
> Regards,
> Christian.
>
> Am 11.01.2017 um 14:51 schrieb Liu, Monk:
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>>
>> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>>
>> And to fully support register reading, we need to implement a new
>> method to use KIQ access register without any schedule() chance, but
>> that's overhead for current stage, and we don't observe registers
>> reading in IRQ context now on 3D apps. (kfd observed such case, and
>> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register
>> dumping, instead of directly reading them via KIQ )
>>    
>> BR Monk
>>
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Wednesday, January 11, 2017 9:38 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Liu, Monk <Monk.Liu@amd.com>
>> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>>
>> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>>> For virtualization, it is must for driver to use KIQ to access
>>> registers when it is out of GPU full access mode.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>>> ---
>>>     drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>>     drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>>     5 files changed, 97 insertions(+), 1 deletion(-)
>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 4185b03..0b8e470 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>>     	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>>     	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>>     	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>>     
>>>     # add asic specific block
>>>     amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o
>>> \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index f82919d..9a2fd3e 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>>     {
>>>     	uint32_t ret;
>>>     
>>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>>> +		return amdgpu_virt_kiq_rreg(adev, reg);
>>> +
>> Mhm, why is the in_interrupt() necessary here?
>>
>>>     	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>     		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>>     	else {
>>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>>     {
>>>     	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>>     
>>> +	if (amdgpu_sriov_runtime(adev))
>>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
>>> +
>> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>>
>> Christian.
>>
>>>     	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>     		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>>     	else {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> new file mode 100644
>>> index 0000000..7861b6b
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -0,0 +1,82 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> +the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
>>> +EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
>>> +USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "amdgpu_virt.h"
>>> +
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>>> +	mutex_init(&adev->virt.lock);
>>> +}
>>> +
>>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>>> +reg) {
>>> +	signed long r;
>>> +	uint32_t val;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_rreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_rreg(ring, reg);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +
>>> +	val = adev->wb.wb[adev->virt.reg_val_offs];
>>> +
>>> +	return val;
>>> +}
>>> +
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v) {
>>> +	signed long r;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_wreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_wreg(ring, reg, v);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 3d38a9f..2869980 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -33,6 +33,7 @@
>>>     struct amdgpu_virt {
>>>     	uint32_t		caps;
>>>     	uint32_t		reg_val_offs;
>>> +	struct mutex		lock;
>>>     };
>>>     
>>>     #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static
>>> inline bool is_virtual_machine(void)
>>>     #endif
>>>     }
>>>     
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
>>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
>>> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v);
>>> +
>>>     #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>>     		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>>     		smc_enabled = true;
>>>     
>>> +	if (amdgpu_sriov_vf(adev))
>>> +		amdgpu_virt_init_setting(adev);
>>> +
>>>     	adev->rev_id = vi_get_rev_id(adev);
>>>     	adev->external_rev_id = 0xFF;
>>>     	switch (adev->asic_type) {
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Yu, Xiangliang]

> -----Original Message-----
> From: Liu, Monk
> Sent: Wednesday, January 11, 2017 10:33 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> See in lines
> 
> 
> -----Original Message-----
> From: Yu, Xiangliang
> Sent: Wednesday, January 11, 2017 10:27 PM
> To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> > This patch is not derived from my work (amd-sriov-4.3/4.6), please
> > don't add my "signed-off-by: Monk Liu " line on it, Seems you add an
> > upper layer interface to invoke on gpu_request/release routines, And I
> > will also take role in the reviewing of it:
> >
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->req_full_gpu(adev, init);
> >
> > [ml]One nitpick , you need place " adev->virt.caps &=
> > ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,
> 
> I think it make more clear as it is not runtime when entering full access mode.
> 
> [ml] NO, you didn't get my point, you should &= ~
> AMDGPU_SRIOV_CAPS_RUNTIME *after* virt->ops->req_full_gpu(adev,
> init)

Good catch.

> 
> >
> > +		r = virt->ops->rel_full_gpu(adev, init);
> > +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> >
> > [ml] what about rel_full_gpu() failed ?  and if rel_full_gpu is
> > designed not allowed to fail then you should type it as non-return
> > function. ( I remember in my first initial implementation on rel_gpu
> > function it is is non-return
> > function)
> 
> I think it is more flexible, we can handle error according to return result.
> 
> [ML] then please handle the error accordingly, but your patch just ignore the
> return value . see it yourself

I have no time to think of the design, maybe later.

> 
> >
> >
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->reset_gpu(adev);
> > +	}
> >
> > [ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME"  should be
> put
> > to the second line as well, like:
> >
> > r = virt-ops->reset_gpu(adev);
> > If (!r)
> >     adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
> >     BUG(); /* we shouldn't fail in requesting GPU_RESET, check
> > hypervisor please */ return r;
> 
> I think we can't hang out VM so rudely, need to print out error message and
> do error handling to enhance the process later.
> 
> [ML] you still not get my point, if reset_gpu() failed. You shouldn't "&=
> ~AMDGPU_SRIOV_CAPS_RUNTIME", but you did it in prior to reset_gpu(),
> isn't it ??
> 
> >
> > BR Monk
> >
> > -----Original Message-----
> > From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> > Sent: Wednesday, January 11, 2017 9:18 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk
> > <Monk.Liu@amd.com>
> > Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for
> > virt
> >
> > Add high level interfaces that is not relate to specific asic. So asic
> > files just need to implement the interfaces to support virtualization.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> > ++++++++++++++++++++++++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
> >  2 files changed, 71 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > index 7861b6b..4e7df8e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
> > *adev, uint32_t reg, uint32_t v)
> >  		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >  	fence_put(f);
> >  }
> > +
> > +/**
> > + * amdgpu_virt_request_full_gpu() - request full gpu access
> > + * @amdgpu:	amdgpu device.
> > + * @init:	is driver init time.
> > + * When start to init/fini driver, first need to request full gpu access.
> > + * Return: Zero if request success, otherwise will return error.
> > + */
> > +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> > +init) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +
> > +	if (virt->ops && virt->ops->req_full_gpu) {
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->req_full_gpu(adev, init);
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/**
> > + * amdgpu_virt_release_full_gpu() - release full gpu access
> > + * @amdgpu:	amdgpu device.
> > + * @init:	is driver init time.
> > + * When finishing driver init/fini, need to release full gpu access.
> > + * Return: Zero if release success, otherwise will returen error.
> > + */
> > +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool
> > +init) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +	int r;
> > +
> > +	if (virt->ops && virt->ops->rel_full_gpu) {
> > +		r = virt->ops->rel_full_gpu(adev, init);
> > +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return r;
> > +	}
> > +	return 0;
> > +}
> > +
> > +/**
> > + * amdgpu_virt_reset_gpu() - reset gpu
> > + * @amdgpu:	amdgpu device.
> > + * Send reset command to GPU hypervisor to reset GPU that VM is using
> > + * Return: Zero if reset success, otherwise will return error.
> > + */
> > +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +
> > +	if (virt->ops && virt->ops->reset_gpu) {
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->reset_gpu(adev);
> > +	}
> > +
> > +	return 0;
> > +}
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 2869980..45771b0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -29,11 +29,22 @@
> >  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> > function */
> >  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU
> is
> > pass through for VM */
> >  #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full
> access
> > mode */
> > +
> > +/**
> > + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
> > +amdgpu_virt_ops {
> > +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> > +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> > +	int (*reset_gpu)(struct amdgpu_device *adev); };
> > +
> >  /* GPU virtualization */
> >  struct amdgpu_virt {
> >  	uint32_t		caps;
> >  	uint32_t		reg_val_offs;
> >  	struct mutex		lock;
> > +	const struct amdgpu_virt_ops	*ops;
> >  };
> >
> >  #define amdgpu_sriov_enabled(adev) \
> > @@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void
> > amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t
> > amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void
> > amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > uint32_t v);
> > +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> > +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
> > +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> >
> >  #endif
> > --
> > 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Liu, Monk]

We will discuss it tomorrow, I'm not sure of your approach right now 

We can stop running driver right after IDH_FLR_NOTIFICATION received and continue driver right after IDH_FLR_NOTIFICATION_CMPL

BR Monk

-----Original Message-----
From: Yu, Xiangliang 
Sent: Wednesday, January 11, 2017 10:43 PM
To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Liu, Shaoyun <Shaoyun.Liu@amd.com>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

> -----Original Message-----
> From: Liu, Monk
> Sent: Wednesday, January 11, 2017 10:28 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd- 
> gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Shaoyun 
> <Shaoyun.Liu@amd.com>
> Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation 
> interfaces
> 
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry) {
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> 
> [ML] we should be able to receive at least two type of IRQ from mailbox:
> One is FLR_NOTIFICATION and another one is IDH_FLR_NOTIFICATION_CMPL,
> 
> The previous one means GIM told us GIM will start do FLR The later one 
> means GIM told us GIM already finished FLR and from that on Guest 
> driver can do re-initialize .
> 
> You only handle the first IRQ, and dispatch a delay_work to monitor 
> event of IDH_FLR_NOTIFICATION_CMPL,
> 
> I suspect that correct scheme may be you handle both of those two type 
> of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

> 
> BR Monk
> 
> -----Original Message-----
> From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> Sent: Wednesday, January 11, 2017 9:18 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk 
> <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
> Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation 
> interfaces
> 
> VI has asic specific virt support, which including mailbox and golden 
> registers init.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> +++++++++++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>  4 files changed, 654 insertions(+), 4 deletions(-)  create mode 
> 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 0b8e470..388e7b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o 
> kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o 
> gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> 
>  amdgpu-y += \
> -	vi.o
> +	vi.o mxgpu_vi.o
> 
>  # add GMC block
>  amdgpu-y += \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 45771b0..a409eec 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
> 
>  /* GPU virtualization */
>  struct amdgpu_virt {
> -	uint32_t		caps;
> -	uint32_t		reg_val_offs;
> -	struct mutex		lock;
> +	uint32_t			caps;
> +	uint32_t			reg_val_offs;
> +	struct mutex			lock;
> +	struct amdgpu_irq_src		ack_irq;
> +	struct amdgpu_irq_src		rcv_irq;
> +	struct delayed_work		flr_work;
>  	const struct amdgpu_virt_ops	*ops;
>  };
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> new file mode 100644
> index 0000000..d2622b6
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> @@ -0,0 +1,592 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person 
> +obtaining a
> + * copy of this software and associated documentation files (the 
> +"Software"),
> + * to deal in the Software without restriction, including without 
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute, 
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom 
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be 
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors: Xiangliang.Yu@amd.com
> + */
> +
> +#include "amdgpu.h"
> +#include "vi.h"
> +#include "bif/bif_5_0_d.h"
> +#include "bif/bif_5_0_sh_mask.h"
> +#include "vid.h"
> +#include "gca/gfx_8_0_d.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "gmc_v8_0.h"
> +#include "gfx_v8_0.h"
> +#include "sdma_v3_0.h"
> +#include "tonga_ih.h"
> +#include "gmc/gmc_8_2_d.h"
> +#include "gmc/gmc_8_2_sh_mask.h"
> +#include "oss/oss_3_0_d.h"
> +#include "oss/oss_3_0_sh_mask.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "dce/dce_10_0_d.h"
> +#include "dce/dce_10_0_sh_mask.h"
> +#include "smu/smu_7_1_3_d.h"
> +#include "mxgpu_vi.h"
> +
> +/* VI golden setting */
> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> +
> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> +
> +static const u32 xgpu_fiji_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> +
> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> +
> +static const u32 xgpu_tonga_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> +	switch (adev->asic_type) {
> +	case CHIP_FIJI:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_settings_a10,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_settings_a10));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_common_all));
> +		break;
> +	case CHIP_TONGA:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_settings_a11,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_settings_a11));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_common_all));
> +		break;
> +	default:
> +		BUG_ON("Doesn't support chip type.\n");
> +		break;
> +	}
> +}
> +
> +/*
> + * Mailbox communication between GPU hypervisor and VFs  */ static 
> +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, 
> +bool
> +val) {
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> +			    TRN_MSG_VALID, val ? 1 : 0);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> +				      enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> +			    MSGBUF_DATA, event);
> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> +
> +	xgpu_vi_mailbox_set_valid(adev, true); }
> +
> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> +				   enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> +	if (reg != event)
> +		return -ENOENT;
> +
> +	/* send ack to PF */
> +	xgpu_vi_mailbox_send_ack(adev);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	while (!(reg & mask)) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		reg = RREG32(mmMAILBOX_CONTROL);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum 
> +idh_event
> +event) {
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +
> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	while (r) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> +					enum idh_request request)
> +{
> +	int r;
> +
> +	xgpu_vi_mailbox_trans_msg(adev, request);
> +
> +	/* start to poll ack */
> +	r = xgpu_vi_poll_ack(adev);
> +	if (r)
> +		return r;
> +
> +	xgpu_vi_mailbox_set_valid(adev, false);
> +
> +	/* start to check msg if request is idh_req_gpu_init_access */
> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> +		if (r)
> +			return r;
> +	}
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> +	return xgpu_vi_send_access_requests(adev,
> IDH_REQ_GPU_RESET_ACCESS); }
> +
> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +
> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> IDH_REQ_GPU_FINI_ACCESS;
> +	return xgpu_vi_send_access_requests(adev, event); }
> +
> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +	int r = 0;
> +
> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> IDH_REL_GPU_FINI_ACCESS;
> +	r = xgpu_vi_send_access_requests(adev, event);
> +
> +	return r;
> +}
> +
> +/* add support mailbox interrupts */
> +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry) {
> +	DRM_DEBUG("get ack intr and do nothing.\n");
> +	return 0;
> +}
> +
> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state) {
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> +	struct amdgpu_virt *virt = container_of(work,
> +					struct amdgpu_virt, flr_work.work);
> +	struct amdgpu_device *adev = container_of(virt,
> +					struct amdgpu_device, virt);
> +	int r = 0;
> +
> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> +	if (r)
> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> +
> +	/* TODO: need to restore gfx states */ }
> +
> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state) {
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry) {
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> +
> +static const struct amdgpu_irq_src_funcs 
> +xgpu_vi_mailbox_ack_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_ack_irq,
> +	.process = xgpu_vi_mailbox_ack_irq,
> +};
> +
> +static const struct amdgpu_irq_src_funcs 
> +xgpu_vi_mailbox_rcv_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> +	.process = xgpu_vi_mailbox_rcv_irq,
> +};
> +
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> +	adev->virt.ack_irq.num_types = 1;
> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> +	adev->virt.rcv_irq.num_types = 1;
> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> +
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> +	int r;
> +
> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> +	if (r)
> +		return r;
> +
> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	return 0;
> +}
> +
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> +	int r;
> +
> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> +	if (r)
> +		return r;
> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> xgpu_vi_mailbox_flr_work);
> +
> +	return 0;
> +}
> +
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> +
> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> +	.reset_gpu		= xgpu_vi_request_reset,
> +};
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> new file mode 100644
> index 0000000..fd6216e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> @@ -0,0 +1,55 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person 
> +obtaining a
> + * copy of this software and associated documentation files (the 
> +"Software"),
> + * to deal in the Software without restriction, including without 
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute, 
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom 
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be 
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef __MXGPU_VI_H__
> +#define __MXGPU_VI_H__
> +
> +#define VI_MAILBOX_TIMEDOUT	150
> +#define VI_MAILBOX_RESET_TIME	12
> +
> +/* VI mailbox messages request */
> +enum idh_request {
> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> +	IDH_REL_GPU_INIT_ACCESS,
> +	IDH_REQ_GPU_FINI_ACCESS,
> +	IDH_REL_GPU_FINI_ACCESS,
> +	IDH_REQ_GPU_RESET_ACCESS
> +};
> +
> +/* VI mailbox messages data */
> +enum idh_event {
> +	IDH_CLR_MSG_BUF = 0,
> +	IDH_READY_TO_ACCESS_GPU,
> +	IDH_FLR_NOTIFICATION,
> +	IDH_FLR_NOTIFICATION_CMPL,
> +	IDH_EVENT_MAX
> +};
> +
> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void 
> +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int 
> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int 
> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void 
> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> +
> +#endif
> --
> 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

We should BUG and not access register at all if found during RUNTIME && IRQ_context
By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:deathsimple@vodafone.de] 
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

> do you think that's okay ?
Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:
> Yeah, make sense
>
> I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?
>
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 10:16 PM
> To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang 
> <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
> That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.
>
> If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.
>
> Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.
>
> Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.
>
> Regards,
> Christian.
>
> Am 11.01.2017 um 14:51 schrieb Liu, Monk:
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>>
>> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>>
>> And to fully support register reading, we need to implement a new 
>> method to use KIQ access register without any schedule() chance, but 
>> that's overhead for current stage, and we don't observe registers 
>> reading in IRQ context now on 3D apps. (kfd observed such case, and 
>> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register 
>> dumping, instead of directly reading them via KIQ )
>>    
>> BR Monk
>>
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Wednesday, January 11, 2017 9:38 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; 
>> amd-gfx@lists.freedesktop.org
>> Cc: Liu, Monk <Monk.Liu@amd.com>
>> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>>
>> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>>> For virtualization, it is must for driver to use KIQ to access 
>>> registers when it is out of GPU full access mode.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>>> ---
>>>     drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>>     drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>>     5 files changed, 97 insertions(+), 1 deletion(-)
>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 4185b03..0b8e470 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>>     	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>>     	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>>     	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>>     
>>>     # add asic specific block
>>>     amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o 
>>> kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index f82919d..9a2fd3e 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>>     {
>>>     	uint32_t ret;
>>>     
>>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>>> +		return amdgpu_virt_kiq_rreg(adev, reg);
>>> +
>> Mhm, why is the in_interrupt() necessary here?
>>
>>>     	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>     		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>>     	else {
>>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>>     {
>>>     	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>>     
>>> +	if (amdgpu_sriov_runtime(adev))
>>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
>>> +
>> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>>
>> Christian.
>>
>>>     	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>     		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>>     	else {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> new file mode 100644
>>> index 0000000..7861b6b
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -0,0 +1,82 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person 
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the 
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without 
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute, 
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to 
>>> +whom the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be 
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO 
>>> +EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
>>> +USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "amdgpu_virt.h"
>>> +
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>>> +	mutex_init(&adev->virt.lock);
>>> +}
>>> +
>>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>>> +reg) {
>>> +	signed long r;
>>> +	uint32_t val;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_rreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_rreg(ring, reg);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +
>>> +	val = adev->wb.wb[adev->virt.reg_val_offs];
>>> +
>>> +	return val;
>>> +}
>>> +
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
>>> +uint32_t v) {
>>> +	signed long r;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_wreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_wreg(ring, reg, v);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 3d38a9f..2869980 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -33,6 +33,7 @@
>>>     struct amdgpu_virt {
>>>     	uint32_t		caps;
>>>     	uint32_t		reg_val_offs;
>>> +	struct mutex		lock;
>>>     };
>>>     
>>>     #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static 
>>> inline bool is_virtual_machine(void)
>>>     #endif
>>>     }
>>>     
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
>>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); 
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
>>> +uint32_t v);
>>> +
>>>     #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c 
>>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>>     		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>>     		smc_enabled = true;
>>>     
>>> +	if (amdgpu_sriov_vf(adev))
>>> +		amdgpu_virt_init_setting(adev);
>>> +
>>>     	adev->rev_id = vi_get_rev_id(adev);
>>>     	adev->external_rev_id = 0xFF;
>>>     	switch (adev->asic_type) {
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> We will discuss it tomorrow, I'm not sure of your approach right now
> 
> We can stop running driver right after IDH_FLR_NOTIFICATION received and
> continue driver right after IDH_FLR_NOTIFICATION_CMPL

I have verified the method on my platform and work fine.
Let's discuss it tomorrow if you have any concern.

> -----Original Message-----
> From: Yu, Xiangliang
> Sent: Wednesday, January 11, 2017 10:43 PM
> To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Liu, Shaoyun <Shaoyun.Liu@amd.com>
> Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> interfaces
> 
> > -----Original Message-----
> > From: Liu, Monk
> > Sent: Wednesday, January 11, 2017 10:28 PM
> > To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> > gfx@lists.freedesktop.org
> > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Shaoyun
> > <Shaoyun.Liu@amd.com>
> > Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> > interfaces
> >
> > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > +				   struct amdgpu_irq_src *source,
> > +				   struct amdgpu_iv_entry *entry) {
> > +	int r;
> > +
> > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > +	/* do nothing for other msg */
> > +	if (r)
> > +		return 0;
> > +
> > +	/* TODO: need to save gfx states */
> > +	schedule_delayed_work(&adev->virt.flr_work,
> > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > +
> > +	return 0;
> > +}
> >
> > [ML] we should be able to receive at least two type of IRQ from mailbox:
> > One is FLR_NOTIFICATION and another one is
> IDH_FLR_NOTIFICATION_CMPL,
> >
> > The previous one means GIM told us GIM will start do FLR The later one
> > means GIM told us GIM already finished FLR and from that on Guest
> > driver can do re-initialize .
> >
> > You only handle the first IRQ, and dispatch a delay_work to monitor
> > event of IDH_FLR_NOTIFICATION_CMPL,
> >
> > I suspect that correct scheme may be you handle both of those two type
> > of IRQ , please check with @Frank and me on the detail tomorrow .
> 
> As I think we may suspend gpu after receive FLR interrupt, polling method is
> safer.
> 
> >
> > BR Monk
> >
> > -----Original Message-----
> > From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> > Sent: Wednesday, January 11, 2017 9:18 PM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk
> > <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
> > Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> > interfaces
> >
> > VI has asic specific virt support, which including mailbox and golden
> > registers init.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
> >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> > +++++++++++++++++++++++++++++++
> >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> >  4 files changed, 654 insertions(+), 4 deletions(-)  create mode
> > 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > index 0b8e470..388e7b0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> cik_ih.o
> > kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o
> gmc_v6_0.o
> > gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> >
> >  amdgpu-y += \
> > -	vi.o
> > +	vi.o mxgpu_vi.o
> >
> >  # add GMC block
> >  amdgpu-y += \
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 45771b0..a409eec 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
> >
> >  /* GPU virtualization */
> >  struct amdgpu_virt {
> > -	uint32_t		caps;
> > -	uint32_t		reg_val_offs;
> > -	struct mutex		lock;
> > +	uint32_t			caps;
> > +	uint32_t			reg_val_offs;
> > +	struct mutex			lock;
> > +	struct amdgpu_irq_src		ack_irq;
> > +	struct amdgpu_irq_src		rcv_irq;
> > +	struct delayed_work		flr_work;
> >  	const struct amdgpu_virt_ops	*ops;
> >  };
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > new file mode 100644
> > index 0000000..d2622b6
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > @@ -0,0 +1,592 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO
> > EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + *
> > + * Authors: Xiangliang.Yu@amd.com
> > + */
> > +
> > +#include "amdgpu.h"
> > +#include "vi.h"
> > +#include "bif/bif_5_0_d.h"
> > +#include "bif/bif_5_0_sh_mask.h"
> > +#include "vid.h"
> > +#include "gca/gfx_8_0_d.h"
> > +#include "gca/gfx_8_0_sh_mask.h"
> > +#include "gmc_v8_0.h"
> > +#include "gfx_v8_0.h"
> > +#include "sdma_v3_0.h"
> > +#include "tonga_ih.h"
> > +#include "gmc/gmc_8_2_d.h"
> > +#include "gmc/gmc_8_2_sh_mask.h"
> > +#include "oss/oss_3_0_d.h"
> > +#include "oss/oss_3_0_sh_mask.h"
> > +#include "gca/gfx_8_0_sh_mask.h"
> > +#include "dce/dce_10_0_d.h"
> > +#include "dce/dce_10_0_sh_mask.h"
> > +#include "smu/smu_7_1_3_d.h"
> > +#include "mxgpu_vi.h"
> > +
> > +/* VI golden setting */
> > +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> > +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> > +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> > +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> > +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> > +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> > +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> > +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> > +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> > +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> > +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> > +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> > +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> > +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> > +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> > +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> > +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> > +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> > +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> > +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> > +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> > +
> > +static const u32 xgpu_fiji_golden_settings_a10[] = {
> > +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > +
> > +static const u32 xgpu_fiji_golden_common_all[] = {
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> > +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> > +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> > +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> > +
> > +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> > +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> > +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> > +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> > +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> > +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> > +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> > +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> > +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> > +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> > +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> > +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> > +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> > +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> > +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> > +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> > +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> > +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> > +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> > +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> > +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> > +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> > +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> > +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> > +};
> > +
> > +static const u32 xgpu_tonga_golden_settings_a11[] = {
> > +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> > +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> > +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> > +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> > +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> > +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > +
> > +static const u32 xgpu_tonga_golden_common_all[] = {
> > +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> > +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> > +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> > +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> > +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> > +
> > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> > +	switch (adev->asic_type) {
> > +	case CHIP_FIJI:
> > +		amdgpu_program_register_sequence(adev,
> > +						 xgpu_fiji_mgcg_cgcg_init,
> > +						 (const u32)ARRAY_SIZE(
> > +						 xgpu_fiji_mgcg_cgcg_init));
> > +		amdgpu_program_register_sequence(adev,
> > +
> > xgpu_fiji_golden_settings_a10,
> > +						 (const u32)ARRAY_SIZE(
> > +
> > xgpu_fiji_golden_settings_a10));
> > +		amdgpu_program_register_sequence(adev,
> > +
> > xgpu_fiji_golden_common_all,
> > +						 (const u32)ARRAY_SIZE(
> > +
> > xgpu_fiji_golden_common_all));
> > +		break;
> > +	case CHIP_TONGA:
> > +		amdgpu_program_register_sequence(adev,
> > +						 xgpu_tonga_mgcg_cgcg_init,
> > +						 (const u32)ARRAY_SIZE(
> > +
> > xgpu_tonga_mgcg_cgcg_init));
> > +		amdgpu_program_register_sequence(adev,
> > +
> > xgpu_tonga_golden_settings_a11,
> > +						 (const u32)ARRAY_SIZE(
> > +
> > xgpu_tonga_golden_settings_a11));
> > +		amdgpu_program_register_sequence(adev,
> > +
> > xgpu_tonga_golden_common_all,
> > +						 (const u32)ARRAY_SIZE(
> > +
> > xgpu_tonga_golden_common_all));
> > +		break;
> > +	default:
> > +		BUG_ON("Doesn't support chip type.\n");
> > +		break;
> > +	}
> > +}
> > +
> > +/*
> > + * Mailbox communication between GPU hypervisor and VFs  */ static
> > +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> > +	WREG32(mmMAILBOX_CONTROL, reg);
> > +}
> > +
> > +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
> > +bool
> > +val) {
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> > +			    TRN_MSG_VALID, val ? 1 : 0);
> > +	WREG32(mmMAILBOX_CONTROL, reg);
> > +}
> > +
> > +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> > +				      enum idh_event event)
> > +{
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> > +			    MSGBUF_DATA, event);
> > +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> > +
> > +	xgpu_vi_mailbox_set_valid(adev, true); }
> > +
> > +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> > +				   enum idh_event event)
> > +{
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> > +	if (reg != event)
> > +		return -ENOENT;
> > +
> > +	/* send ack to PF */
> > +	xgpu_vi_mailbox_send_ack(adev);
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	while (!(reg & mask)) {
> > +		if (timeout <= 0) {
> > +			pr_err("Doesn't get ack from pf.\n");
> > +			r = -ETIME;
> > +			break;
> > +		}
> > +		msleep(1);
> > +		timeout -= 1;
> > +
> > +		reg = RREG32(mmMAILBOX_CONTROL);
> > +	}
> > +
> > +	return r;
> > +}
> > +
> > +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
> > +idh_event
> > +event) {
> > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > +
> > +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > +	while (r) {
> > +		if (timeout <= 0) {
> > +			pr_err("Doesn't get ack from pf.\n");
> > +			r = -ETIME;
> > +			break;
> > +		}
> > +		msleep(1);
> > +		timeout -= 1;
> > +
> > +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > +	}
> > +
> > +	return r;
> > +}
> > +
> > +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> > +					enum idh_request request)
> > +{
> > +	int r;
> > +
> > +	xgpu_vi_mailbox_trans_msg(adev, request);
> > +
> > +	/* start to poll ack */
> > +	r = xgpu_vi_poll_ack(adev);
> > +	if (r)
> > +		return r;
> > +
> > +	xgpu_vi_mailbox_set_valid(adev, false);
> > +
> > +	/* start to check msg if request is idh_req_gpu_init_access */
> > +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> > +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> > +		if (r)
> > +			return r;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> > +	return xgpu_vi_send_access_requests(adev,
> > IDH_REQ_GPU_RESET_ACCESS); }
> > +
> > +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> > +					   bool init)
> > +{
> > +	enum idh_event event;
> > +
> > +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> > IDH_REQ_GPU_FINI_ACCESS;
> > +	return xgpu_vi_send_access_requests(adev, event); }
> > +
> > +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> > +					   bool init)
> > +{
> > +	enum idh_event event;
> > +	int r = 0;
> > +
> > +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> > IDH_REL_GPU_FINI_ACCESS;
> > +	r = xgpu_vi_send_access_requests(adev, event);
> > +
> > +	return r;
> > +}
> > +
> > +/* add support mailbox interrupts */
> > +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> > +				   struct amdgpu_irq_src *source,
> > +				   struct amdgpu_iv_entry *entry) {
> > +	DRM_DEBUG("get ack intr and do nothing.\n");
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> > +				       struct amdgpu_irq_src *src,
> > +				       unsigned type,
> > +				       enum amdgpu_interrupt_state state) {
> > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > +
> > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > +
> > +	return 0;
> > +}
> > +
> > +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> > +	struct amdgpu_virt *virt = container_of(work,
> > +					struct amdgpu_virt, flr_work.work);
> > +	struct amdgpu_device *adev = container_of(virt,
> > +					struct amdgpu_device, virt);
> > +	int r = 0;
> > +
> > +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> > +	if (r)
> > +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> > +
> > +	/* TODO: need to restore gfx states */ }
> > +
> > +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> > +				       struct amdgpu_irq_src *src,
> > +				       unsigned type,
> > +				       enum amdgpu_interrupt_state state) {
> > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > +
> > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > +				   struct amdgpu_irq_src *source,
> > +				   struct amdgpu_iv_entry *entry) {
> > +	int r;
> > +
> > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > +	/* do nothing for other msg */
> > +	if (r)
> > +		return 0;
> > +
> > +	/* TODO: need to save gfx states */
> > +	schedule_delayed_work(&adev->virt.flr_work,
> > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > +
> > +	return 0;
> > +}
> > +
> > +static const struct amdgpu_irq_src_funcs
> > +xgpu_vi_mailbox_ack_irq_funcs =
> > {
> > +	.set = xgpu_vi_set_mailbox_ack_irq,
> > +	.process = xgpu_vi_mailbox_ack_irq,
> > +};
> > +
> > +static const struct amdgpu_irq_src_funcs
> > +xgpu_vi_mailbox_rcv_irq_funcs =
> > {
> > +	.set = xgpu_vi_set_mailbox_rcv_irq,
> > +	.process = xgpu_vi_mailbox_rcv_irq,
> > +};
> > +
> > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> > +	adev->virt.ack_irq.num_types = 1;
> > +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> > +	adev->virt.rcv_irq.num_types = 1;
> > +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> > +
> > +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> > +	int r;
> > +
> > +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> > +	if (r)
> > +		return r;
> > +
> > +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> > +	if (r) {
> > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > +		return r;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> > +	int r;
> > +
> > +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> > +	if (r)
> > +		return r;
> > +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> > +	if (r) {
> > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > +		return r;
> > +	}
> > +
> > +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> > xgpu_vi_mailbox_flr_work);
> > +
> > +	return 0;
> > +}
> > +
> > +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> > +	cancel_delayed_work_sync(&adev->virt.flr_work);
> > +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> > +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> > +
> > +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> > +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> > +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> > +	.reset_gpu		= xgpu_vi_request_reset,
> > +};
> > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > new file mode 100644
> > index 0000000..fd6216e
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > @@ -0,0 +1,55 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO
> > EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + */
> > +
> > +#ifndef __MXGPU_VI_H__
> > +#define __MXGPU_VI_H__
> > +
> > +#define VI_MAILBOX_TIMEDOUT	150
> > +#define VI_MAILBOX_RESET_TIME	12
> > +
> > +/* VI mailbox messages request */
> > +enum idh_request {
> > +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> > +	IDH_REL_GPU_INIT_ACCESS,
> > +	IDH_REQ_GPU_FINI_ACCESS,
> > +	IDH_REL_GPU_FINI_ACCESS,
> > +	IDH_REQ_GPU_RESET_ACCESS
> > +};
> > +
> > +/* VI mailbox messages data */
> > +enum idh_event {
> > +	IDH_CLR_MSG_BUF = 0,
> > +	IDH_READY_TO_ACCESS_GPU,
> > +	IDH_FLR_NOTIFICATION,
> > +	IDH_FLR_NOTIFICATION_CMPL,
> > +	IDH_EVENT_MAX
> > +};
> > +
> > +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> > +
> > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
> > +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
> > +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> > +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> > +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> > +
> > +#endif
> > --
> > 2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> > We will discuss it tomorrow, I'm not sure of your approach right now
> >
> > We can stop running driver right after IDH_FLR_NOTIFICATION received
> > and continue driver right after IDH_FLR_NOTIFICATION_CMPL

If missing the interrupt, I think we can't easy to handle the error. That what i want to avoid it.

> 
> > -----Original Message-----
> > From: Yu, Xiangliang
> > Sent: Wednesday, January 11, 2017 10:43 PM
> > To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
> > Cc: Liu, Shaoyun <Shaoyun.Liu@amd.com>
> > Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> > interfaces
> >
> > > -----Original Message-----
> > > From: Liu, Monk
> > > Sent: Wednesday, January 11, 2017 10:28 PM
> > > To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> > > gfx@lists.freedesktop.org
> > > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Shaoyun
> > > <Shaoyun.Liu@amd.com>
> > > Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> > > interfaces
> > >
> > > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	int r;
> > > +
> > > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > > +	/* do nothing for other msg */
> > > +	if (r)
> > > +		return 0;
> > > +
> > > +	/* TODO: need to save gfx states */
> > > +	schedule_delayed_work(&adev->virt.flr_work,
> > > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > > +
> > > +	return 0;
> > > +}
> > >
> > > [ML] we should be able to receive at least two type of IRQ from mailbox:
> > > One is FLR_NOTIFICATION and another one is
> > IDH_FLR_NOTIFICATION_CMPL,
> > >
> > > The previous one means GIM told us GIM will start do FLR The later
> > > one means GIM told us GIM already finished FLR and from that on
> > > Guest driver can do re-initialize .
> > >
> > > You only handle the first IRQ, and dispatch a delay_work to monitor
> > > event of IDH_FLR_NOTIFICATION_CMPL,
> > >
> > > I suspect that correct scheme may be you handle both of those two
> > > type of IRQ , please check with @Frank and me on the detail tomorrow .
> >
> > As I think we may suspend gpu after receive FLR interrupt, polling
> > method is safer.
> >
> > >
> > > BR Monk
> > >
> > > -----Original Message-----
> > > From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> > > Sent: Wednesday, January 11, 2017 9:18 PM
> > > To: amd-gfx@lists.freedesktop.org
> > > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk
> > > <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
> > > Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation
> > > interfaces
> > >
> > > VI has asic specific virt support, which including mailbox and
> > > golden registers init.
> > >
> > > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > > Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
> > > ---
> > >  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> > >  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
> > >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> > > +++++++++++++++++++++++++++++++
> > >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> > >  4 files changed, 654 insertions(+), 4 deletions(-)  create mode
> > > 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > >  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > > index 0b8e470..388e7b0 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > > @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> > cik_ih.o
> > > kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o
> > gmc_v6_0.o
> > > gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> > >
> > >  amdgpu-y += \
> > > -	vi.o
> > > +	vi.o mxgpu_vi.o
> > >
> > >  # add GMC block
> > >  amdgpu-y += \
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > index 45771b0..a409eec 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > @@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
> > >
> > >  /* GPU virtualization */
> > >  struct amdgpu_virt {
> > > -	uint32_t		caps;
> > > -	uint32_t		reg_val_offs;
> > > -	struct mutex		lock;
> > > +	uint32_t			caps;
> > > +	uint32_t			reg_val_offs;
> > > +	struct mutex			lock;
> > > +	struct amdgpu_irq_src		ack_irq;
> > > +	struct amdgpu_irq_src		rcv_irq;
> > > +	struct delayed_work		flr_work;
> > >  	const struct amdgpu_virt_ops	*ops;
> > >  };
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > new file mode 100644
> > > index 0000000..d2622b6
> > > --- /dev/null
> > > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > @@ -0,0 +1,592 @@
> > > +/*
> > > + * Copyright 2017 Advanced Micro Devices, Inc.
> > > + *
> > > + * Permission is hereby granted, free of charge, to any person
> > > +obtaining a
> > > + * copy of this software and associated documentation files (the
> > > +"Software"),
> > > + * to deal in the Software without restriction, including without
> > > +limitation
> > > + * the rights to use, copy, modify, merge, publish, distribute,
> > > +sublicense,
> > > + * and/or sell copies of the Software, and to permit persons to
> > > +whom the
> > > + * Software is furnished to do so, subject to the following conditions:
> > > + *
> > > + * The above copyright notice and this permission notice shall be
> > > +included in
> > > + * all copies or substantial portions of the Software.
> > > + *
> > > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> > KIND,
> > > +EXPRESS OR
> > > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > > +MERCHANTABILITY,
> > > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> > NO
> > > EVENT
> > > +SHALL
> > > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > > +DAMAGES OR
> > > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > > +OTHERWISE,
> > > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > > THE USE
> > > +OR
> > > + * OTHER DEALINGS IN THE SOFTWARE.
> > > + *
> > > + * Authors: Xiangliang.Yu@amd.com
> > > + */
> > > +
> > > +#include "amdgpu.h"
> > > +#include "vi.h"
> > > +#include "bif/bif_5_0_d.h"
> > > +#include "bif/bif_5_0_sh_mask.h"
> > > +#include "vid.h"
> > > +#include "gca/gfx_8_0_d.h"
> > > +#include "gca/gfx_8_0_sh_mask.h"
> > > +#include "gmc_v8_0.h"
> > > +#include "gfx_v8_0.h"
> > > +#include "sdma_v3_0.h"
> > > +#include "tonga_ih.h"
> > > +#include "gmc/gmc_8_2_d.h"
> > > +#include "gmc/gmc_8_2_sh_mask.h"
> > > +#include "oss/oss_3_0_d.h"
> > > +#include "oss/oss_3_0_sh_mask.h"
> > > +#include "gca/gfx_8_0_sh_mask.h"
> > > +#include "dce/dce_10_0_d.h"
> > > +#include "dce/dce_10_0_sh_mask.h"
> > > +#include "smu/smu_7_1_3_d.h"
> > > +#include "mxgpu_vi.h"
> > > +
> > > +/* VI golden setting */
> > > +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> > > +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> > > +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> > > +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> > > +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> > > +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> > > +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> > > +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> > > +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> > > +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> > > +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> > > +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> > > +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> > > +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> > > +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> > > +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> > > +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> > > +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> > > +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> > > +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> > > +
> > > +static const u32 xgpu_fiji_golden_settings_a10[] = {
> > > +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> > > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> > > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > > +
> > > +static const u32 xgpu_fiji_golden_common_all[] = {
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> > > +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> > > +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> > > +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> > > +
> > > +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> > > +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> > > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > > +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> > > +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> > > +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> > > +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> > > +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> > > +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> > > +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> > > +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> > > +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > > +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > > +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> > > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > > +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > > +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > > +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> > > +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> > > +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> > > +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> > > +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> > > +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> > > +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> > > +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> > > +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> > > +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> > > +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> > > +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> > > +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> > > +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> > > +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> > > +};
> > > +
> > > +static const u32 xgpu_tonga_golden_settings_a11[] = {
> > > +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> > > +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> > > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > > +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> > > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > > +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> > > +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> > > +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> > > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > > +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> > > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > > +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> > > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> > > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> > > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > > +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> > > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> > > +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> > > +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> > > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > > +
> > > +static const u32 xgpu_tonga_golden_common_all[] = {
> > > +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> > > +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> > > +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> > > +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> > > +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> > > +
> > > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> > > +	switch (adev->asic_type) {
> > > +	case CHIP_FIJI:
> > > +		amdgpu_program_register_sequence(adev,
> > > +						 xgpu_fiji_mgcg_cgcg_init,
> > > +						 (const u32)ARRAY_SIZE(
> > > +						 xgpu_fiji_mgcg_cgcg_init));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_fiji_golden_settings_a10,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_fiji_golden_settings_a10));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_fiji_golden_common_all,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_fiji_golden_common_all));
> > > +		break;
> > > +	case CHIP_TONGA:
> > > +		amdgpu_program_register_sequence(adev,
> > > +						 xgpu_tonga_mgcg_cgcg_init,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_mgcg_cgcg_init));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_tonga_golden_settings_a11,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_golden_settings_a11));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_tonga_golden_common_all,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_golden_common_all));
> > > +		break;
> > > +	default:
> > > +		BUG_ON("Doesn't support chip type.\n");
> > > +		break;
> > > +	}
> > > +}
> > > +
> > > +/*
> > > + * Mailbox communication between GPU hypervisor and VFs  */ static
> > > +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> > > +	WREG32(mmMAILBOX_CONTROL, reg);
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
> > > +bool
> > > +val) {
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> > > +			    TRN_MSG_VALID, val ? 1 : 0);
> > > +	WREG32(mmMAILBOX_CONTROL, reg);
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> > > +				      enum idh_event event)
> > > +{
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> > > +			    MSGBUF_DATA, event);
> > > +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> > > +
> > > +	xgpu_vi_mailbox_set_valid(adev, true); }
> > > +
> > > +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> > > +				   enum idh_event event)
> > > +{
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> > > +	if (reg != event)
> > > +		return -ENOENT;
> > > +
> > > +	/* send ack to PF */
> > > +	xgpu_vi_mailbox_send_ack(adev);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> > > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > > +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	while (!(reg & mask)) {
> > > +		if (timeout <= 0) {
> > > +			pr_err("Doesn't get ack from pf.\n");
> > > +			r = -ETIME;
> > > +			break;
> > > +		}
> > > +		msleep(1);
> > > +		timeout -= 1;
> > > +
> > > +		reg = RREG32(mmMAILBOX_CONTROL);
> > > +	}
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
> > > +idh_event
> > > +event) {
> > > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > > +
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > > +	while (r) {
> > > +		if (timeout <= 0) {
> > > +			pr_err("Doesn't get ack from pf.\n");
> > > +			r = -ETIME;
> > > +			break;
> > > +		}
> > > +		msleep(1);
> > > +		timeout -= 1;
> > > +
> > > +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > > +	}
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> > > +					enum idh_request request)
> > > +{
> > > +	int r;
> > > +
> > > +	xgpu_vi_mailbox_trans_msg(adev, request);
> > > +
> > > +	/* start to poll ack */
> > > +	r = xgpu_vi_poll_ack(adev);
> > > +	if (r)
> > > +		return r;
> > > +
> > > +	xgpu_vi_mailbox_set_valid(adev, false);
> > > +
> > > +	/* start to check msg if request is idh_req_gpu_init_access */
> > > +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> > > +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> > > +		if (r)
> > > +			return r;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> > > +	return xgpu_vi_send_access_requests(adev,
> > > IDH_REQ_GPU_RESET_ACCESS); }
> > > +
> > > +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device
> *adev,
> > > +					   bool init)
> > > +{
> > > +	enum idh_event event;
> > > +
> > > +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> > > IDH_REQ_GPU_FINI_ACCESS;
> > > +	return xgpu_vi_send_access_requests(adev, event); }
> > > +
> > > +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device
> *adev,
> > > +					   bool init)
> > > +{
> > > +	enum idh_event event;
> > > +	int r = 0;
> > > +
> > > +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> > > IDH_REL_GPU_FINI_ACCESS;
> > > +	r = xgpu_vi_send_access_requests(adev, event);
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +/* add support mailbox interrupts */ static int
> > > +xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	DRM_DEBUG("get ack intr and do nothing.\n");
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> > > +				       struct amdgpu_irq_src *src,
> > > +				       unsigned type,
> > > +				       enum amdgpu_interrupt_state state) {
> > > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > > +
> > > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> > > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> > > +	struct amdgpu_virt *virt = container_of(work,
> > > +					struct amdgpu_virt, flr_work.work);
> > > +	struct amdgpu_device *adev = container_of(virt,
> > > +					struct amdgpu_device, virt);
> > > +	int r = 0;
> > > +
> > > +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> > > +	if (r)
> > > +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> > > +
> > > +	/* TODO: need to restore gfx states */ }
> > > +
> > > +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				       struct amdgpu_irq_src *src,
> > > +				       unsigned type,
> > > +				       enum amdgpu_interrupt_state state) {
> > > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > > +
> > > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> > > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	int r;
> > > +
> > > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > > +	/* do nothing for other msg */
> > > +	if (r)
> > > +		return 0;
> > > +
> > > +	/* TODO: need to save gfx states */
> > > +	schedule_delayed_work(&adev->virt.flr_work,
> > > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static const struct amdgpu_irq_src_funcs
> > > +xgpu_vi_mailbox_ack_irq_funcs =
> > > {
> > > +	.set = xgpu_vi_set_mailbox_ack_irq,
> > > +	.process = xgpu_vi_mailbox_ack_irq, };
> > > +
> > > +static const struct amdgpu_irq_src_funcs
> > > +xgpu_vi_mailbox_rcv_irq_funcs =
> > > {
> > > +	.set = xgpu_vi_set_mailbox_rcv_irq,
> > > +	.process = xgpu_vi_mailbox_rcv_irq, };
> > > +
> > > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> > > +	adev->virt.ack_irq.num_types = 1;
> > > +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> > > +	adev->virt.rcv_irq.num_types = 1;
> > > +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> > > +
> > > +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> > > +	int r;
> > > +
> > > +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> > > +	if (r)
> > > +		return r;
> > > +
> > > +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> > > +	if (r) {
> > > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > > +		return r;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> > > +	int r;
> > > +
> > > +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> > > +	if (r)
> > > +		return r;
> > > +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> > > +	if (r) {
> > > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > > +		return r;
> > > +	}
> > > +
> > > +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> > > xgpu_vi_mailbox_flr_work);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> > > +	cancel_delayed_work_sync(&adev->virt.flr_work);
> > > +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> > > +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> > > +
> > > +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> > > +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> > > +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> > > +	.reset_gpu		= xgpu_vi_request_reset,
> > > +};
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > new file mode 100644
> > > index 0000000..fd6216e
> > > --- /dev/null
> > > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > @@ -0,0 +1,55 @@
> > > +/*
> > > + * Copyright 2017 Advanced Micro Devices, Inc.
> > > + *
> > > + * Permission is hereby granted, free of charge, to any person
> > > +obtaining a
> > > + * copy of this software and associated documentation files (the
> > > +"Software"),
> > > + * to deal in the Software without restriction, including without
> > > +limitation
> > > + * the rights to use, copy, modify, merge, publish, distribute,
> > > +sublicense,
> > > + * and/or sell copies of the Software, and to permit persons to
> > > +whom the
> > > + * Software is furnished to do so, subject to the following conditions:
> > > + *
> > > + * The above copyright notice and this permission notice shall be
> > > +included in
> > > + * all copies or substantial portions of the Software.
> > > + *
> > > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> > KIND,
> > > +EXPRESS OR
> > > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > > +MERCHANTABILITY,
> > > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> > NO
> > > EVENT
> > > +SHALL
> > > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > > +DAMAGES OR
> > > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > > +OTHERWISE,
> > > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > > THE USE
> > > +OR
> > > + * OTHER DEALINGS IN THE SOFTWARE.
> > > + */
> > > +
> > > +#ifndef __MXGPU_VI_H__
> > > +#define __MXGPU_VI_H__
> > > +
> > > +#define VI_MAILBOX_TIMEDOUT	150
> > > +#define VI_MAILBOX_RESET_TIME	12
> > > +
> > > +/* VI mailbox messages request */
> > > +enum idh_request {
> > > +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> > > +	IDH_REL_GPU_INIT_ACCESS,
> > > +	IDH_REQ_GPU_FINI_ACCESS,
> > > +	IDH_REL_GPU_FINI_ACCESS,
> > > +	IDH_REQ_GPU_RESET_ACCESS
> > > +};
> > > +
> > > +/* VI mailbox messages data */
> > > +enum idh_event {
> > > +	IDH_CLR_MSG_BUF = 0,
> > > +	IDH_READY_TO_ACCESS_GPU,
> > > +	IDH_FLR_NOTIFICATION,
> > > +	IDH_FLR_NOTIFICATION_CMPL,
> > > +	IDH_EVENT_MAX
> > > +};
> > > +
> > > +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> > > +
> > > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
> > > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
> > > +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> > > +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> > > +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> > > +
> > > +#endif
> > > --
> > > 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Liu, Monk]

Sounds make sense!
BR Monk

-----Original Message-----
From: Yu, Xiangliang 
Sent: Wednesday, January 11, 2017 11:04 PM
To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Liu, Shaoyun <Shaoyun.Liu@amd.com>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

> > We will discuss it tomorrow, I'm not sure of your approach right now
> >
> > We can stop running driver right after IDH_FLR_NOTIFICATION received 
> > and continue driver right after IDH_FLR_NOTIFICATION_CMPL

If missing the interrupt, I think we can't easy to handle the error. That what i want to avoid it.

> 
> > -----Original Message-----
> > From: Yu, Xiangliang
> > Sent: Wednesday, January 11, 2017 10:43 PM
> > To: Liu, Monk <Monk.Liu@amd.com>; amd-gfx@lists.freedesktop.org
> > Cc: Liu, Shaoyun <Shaoyun.Liu@amd.com>
> > Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation 
> > interfaces
> >
> > > -----Original Message-----
> > > From: Liu, Monk
> > > Sent: Wednesday, January 11, 2017 10:28 PM
> > > To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd- 
> > > gfx@lists.freedesktop.org
> > > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Shaoyun 
> > > <Shaoyun.Liu@amd.com>
> > > Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt 
> > > operation interfaces
> > >
> > > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	int r;
> > > +
> > > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > > +	/* do nothing for other msg */
> > > +	if (r)
> > > +		return 0;
> > > +
> > > +	/* TODO: need to save gfx states */
> > > +	schedule_delayed_work(&adev->virt.flr_work,
> > > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > > +
> > > +	return 0;
> > > +}
> > >
> > > [ML] we should be able to receive at least two type of IRQ from mailbox:
> > > One is FLR_NOTIFICATION and another one is
> > IDH_FLR_NOTIFICATION_CMPL,
> > >
> > > The previous one means GIM told us GIM will start do FLR The later 
> > > one means GIM told us GIM already finished FLR and from that on 
> > > Guest driver can do re-initialize .
> > >
> > > You only handle the first IRQ, and dispatch a delay_work to 
> > > monitor event of IDH_FLR_NOTIFICATION_CMPL,
> > >
> > > I suspect that correct scheme may be you handle both of those two 
> > > type of IRQ , please check with @Frank and me on the detail tomorrow .
> >
> > As I think we may suspend gpu after receive FLR interrupt, polling 
> > method is safer.
> >
> > >
> > > BR Monk
> > >
> > > -----Original Message-----
> > > From: Xiangliang Yu [mailto:Xiangliang.Yu@amd.com]
> > > Sent: Wednesday, January 11, 2017 9:18 PM
> > > To: amd-gfx@lists.freedesktop.org
> > > Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>; Liu, Monk 
> > > <Monk.Liu@amd.com>; Liu, Shaoyun <Shaoyun.Liu@amd.com>
> > > Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation 
> > > interfaces
> > >
> > > VI has asic specific virt support, which including mailbox and 
> > > golden registers init.
> > >
> > > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > > Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
> > > ---
> > >  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> > >  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
> > >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> > > +++++++++++++++++++++++++++++++
> > >  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> > >  4 files changed, 654 insertions(+), 4 deletions(-)  create mode
> > > 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > >  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > > index 0b8e470..388e7b0 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > > @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> > cik_ih.o
> > > kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o
> > gmc_v6_0.o
> > > gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> > >
> > >  amdgpu-y += \
> > > -	vi.o
> > > +	vi.o mxgpu_vi.o
> > >
> > >  # add GMC block
> > >  amdgpu-y += \
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > index 45771b0..a409eec 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > > @@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
> > >
> > >  /* GPU virtualization */
> > >  struct amdgpu_virt {
> > > -	uint32_t		caps;
> > > -	uint32_t		reg_val_offs;
> > > -	struct mutex		lock;
> > > +	uint32_t			caps;
> > > +	uint32_t			reg_val_offs;
> > > +	struct mutex			lock;
> > > +	struct amdgpu_irq_src		ack_irq;
> > > +	struct amdgpu_irq_src		rcv_irq;
> > > +	struct delayed_work		flr_work;
> > >  	const struct amdgpu_virt_ops	*ops;
> > >  };
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > new file mode 100644
> > > index 0000000..d2622b6
> > > --- /dev/null
> > > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > > @@ -0,0 +1,592 @@
> > > +/*
> > > + * Copyright 2017 Advanced Micro Devices, Inc.
> > > + *
> > > + * Permission is hereby granted, free of charge, to any person 
> > > +obtaining a
> > > + * copy of this software and associated documentation files (the 
> > > +"Software"),
> > > + * to deal in the Software without restriction, including without 
> > > +limitation
> > > + * the rights to use, copy, modify, merge, publish, distribute, 
> > > +sublicense,
> > > + * and/or sell copies of the Software, and to permit persons to 
> > > +whom the
> > > + * Software is furnished to do so, subject to the following conditions:
> > > + *
> > > + * The above copyright notice and this permission notice shall be 
> > > +included in
> > > + * all copies or substantial portions of the Software.
> > > + *
> > > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> > KIND,
> > > +EXPRESS OR
> > > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> > > +MERCHANTABILITY,
> > > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> > NO
> > > EVENT
> > > +SHALL
> > > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> > > +DAMAGES OR
> > > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> > > +OTHERWISE,
> > > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > > THE USE
> > > +OR
> > > + * OTHER DEALINGS IN THE SOFTWARE.
> > > + *
> > > + * Authors: Xiangliang.Yu@amd.com  */
> > > +
> > > +#include "amdgpu.h"
> > > +#include "vi.h"
> > > +#include "bif/bif_5_0_d.h"
> > > +#include "bif/bif_5_0_sh_mask.h"
> > > +#include "vid.h"
> > > +#include "gca/gfx_8_0_d.h"
> > > +#include "gca/gfx_8_0_sh_mask.h"
> > > +#include "gmc_v8_0.h"
> > > +#include "gfx_v8_0.h"
> > > +#include "sdma_v3_0.h"
> > > +#include "tonga_ih.h"
> > > +#include "gmc/gmc_8_2_d.h"
> > > +#include "gmc/gmc_8_2_sh_mask.h"
> > > +#include "oss/oss_3_0_d.h"
> > > +#include "oss/oss_3_0_sh_mask.h"
> > > +#include "gca/gfx_8_0_sh_mask.h"
> > > +#include "dce/dce_10_0_d.h"
> > > +#include "dce/dce_10_0_sh_mask.h"
> > > +#include "smu/smu_7_1_3_d.h"
> > > +#include "mxgpu_vi.h"
> > > +
> > > +/* VI golden setting */
> > > +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> > > +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> > > +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> > > +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> > > +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> > > +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> > > +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > > +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> > > +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> > > +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> > > +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> > > +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> > > +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> > > +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> > > +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> > > +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> > > +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> > > +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> > > +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> > > +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> > > +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> > > +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> > > +
> > > +static const u32 xgpu_fiji_golden_settings_a10[] = {
> > > +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> > > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> > > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > > +
> > > +static const u32 xgpu_fiji_golden_common_all[] = {
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> > > +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> > > +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> > > +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> > > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > > +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> > > +
> > > +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> > > +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> > > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > > +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> > > +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> > > +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> > > +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> > > +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> > > +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> > > +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> > > +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> > > +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> > > +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> > > +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> > > +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > > +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > > +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> > > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > > +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > > +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > > +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > > +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > > +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > > +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > > +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > > +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> > > +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> > > +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> > > +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> > > +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> > > +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> > > +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> > > +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> > > +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> > > +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> > > +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> > > +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> > > +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> > > +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> > > +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> > > +};
> > > +
> > > +static const u32 xgpu_tonga_golden_settings_a11[] = {
> > > +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> > > +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> > > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > > +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> > > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > > +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> > > +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> > > +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> > > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > > +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> > > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > > +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> > > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> > > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > > +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> > > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > > +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> > > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> > > +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> > > +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> > > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > > +
> > > +static const u32 xgpu_tonga_golden_common_all[] = {
> > > +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> > > +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> > > +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> > > +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> > > +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> > > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> > > +
> > > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> > > +	switch (adev->asic_type) {
> > > +	case CHIP_FIJI:
> > > +		amdgpu_program_register_sequence(adev,
> > > +						 xgpu_fiji_mgcg_cgcg_init,
> > > +						 (const u32)ARRAY_SIZE(
> > > +						 xgpu_fiji_mgcg_cgcg_init));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_fiji_golden_settings_a10,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_fiji_golden_settings_a10));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_fiji_golden_common_all,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_fiji_golden_common_all));
> > > +		break;
> > > +	case CHIP_TONGA:
> > > +		amdgpu_program_register_sequence(adev,
> > > +						 xgpu_tonga_mgcg_cgcg_init,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_mgcg_cgcg_init));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_tonga_golden_settings_a11,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_golden_settings_a11));
> > > +		amdgpu_program_register_sequence(adev,
> > > +
> > > xgpu_tonga_golden_common_all,
> > > +						 (const u32)ARRAY_SIZE(
> > > +
> > > xgpu_tonga_golden_common_all));
> > > +		break;
> > > +	default:
> > > +		BUG_ON("Doesn't support chip type.\n");
> > > +		break;
> > > +	}
> > > +}
> > > +
> > > +/*
> > > + * Mailbox communication between GPU hypervisor and VFs  */ 
> > > +static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> > > +	WREG32(mmMAILBOX_CONTROL, reg);
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, 
> > > +bool
> > > +val) {
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> > > +			    TRN_MSG_VALID, val ? 1 : 0);
> > > +	WREG32(mmMAILBOX_CONTROL, reg);
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> > > +				      enum idh_event event)
> > > +{
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> > > +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> > > +			    MSGBUF_DATA, event);
> > > +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> > > +
> > > +	xgpu_vi_mailbox_set_valid(adev, true); }
> > > +
> > > +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> > > +				   enum idh_event event)
> > > +{
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> > > +	if (reg != event)
> > > +		return -ENOENT;
> > > +
> > > +	/* send ack to PF */
> > > +	xgpu_vi_mailbox_send_ack(adev);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> > > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > > +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> > > +	u32 reg;
> > > +
> > > +	reg = RREG32(mmMAILBOX_CONTROL);
> > > +	while (!(reg & mask)) {
> > > +		if (timeout <= 0) {
> > > +			pr_err("Doesn't get ack from pf.\n");
> > > +			r = -ETIME;
> > > +			break;
> > > +		}
> > > +		msleep(1);
> > > +		timeout -= 1;
> > > +
> > > +		reg = RREG32(mmMAILBOX_CONTROL);
> > > +	}
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum 
> > > +idh_event
> > > +event) {
> > > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > > +
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > > +	while (r) {
> > > +		if (timeout <= 0) {
> > > +			pr_err("Doesn't get ack from pf.\n");
> > > +			r = -ETIME;
> > > +			break;
> > > +		}
> > > +		msleep(1);
> > > +		timeout -= 1;
> > > +
> > > +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > > +	}
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> > > +					enum idh_request request)
> > > +{
> > > +	int r;
> > > +
> > > +	xgpu_vi_mailbox_trans_msg(adev, request);
> > > +
> > > +	/* start to poll ack */
> > > +	r = xgpu_vi_poll_ack(adev);
> > > +	if (r)
> > > +		return r;
> > > +
> > > +	xgpu_vi_mailbox_set_valid(adev, false);
> > > +
> > > +	/* start to check msg if request is idh_req_gpu_init_access */
> > > +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> > > +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> > > +		if (r)
> > > +			return r;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> > > +	return xgpu_vi_send_access_requests(adev,
> > > IDH_REQ_GPU_RESET_ACCESS); }
> > > +
> > > +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device
> *adev,
> > > +					   bool init)
> > > +{
> > > +	enum idh_event event;
> > > +
> > > +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> > > IDH_REQ_GPU_FINI_ACCESS;
> > > +	return xgpu_vi_send_access_requests(adev, event); }
> > > +
> > > +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device
> *adev,
> > > +					   bool init)
> > > +{
> > > +	enum idh_event event;
> > > +	int r = 0;
> > > +
> > > +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> > > IDH_REL_GPU_FINI_ACCESS;
> > > +	r = xgpu_vi_send_access_requests(adev, event);
> > > +
> > > +	return r;
> > > +}
> > > +
> > > +/* add support mailbox interrupts */ static int 
> > > +xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	DRM_DEBUG("get ack intr and do nothing.\n");
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> > > +				       struct amdgpu_irq_src *src,
> > > +				       unsigned type,
> > > +				       enum amdgpu_interrupt_state state) {
> > > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > > +
> > > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> > > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> > > +	struct amdgpu_virt *virt = container_of(work,
> > > +					struct amdgpu_virt, flr_work.work);
> > > +	struct amdgpu_device *adev = container_of(virt,
> > > +					struct amdgpu_device, virt);
> > > +	int r = 0;
> > > +
> > > +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> > > +	if (r)
> > > +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> > > +
> > > +	/* TODO: need to restore gfx states */ }
> > > +
> > > +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				       struct amdgpu_irq_src *src,
> > > +				       unsigned type,
> > > +				       enum amdgpu_interrupt_state state) {
> > > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > > +
> > > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> > > +			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> > > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > > +				   struct amdgpu_irq_src *source,
> > > +				   struct amdgpu_iv_entry *entry) {
> > > +	int r;
> > > +
> > > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > > +	/* do nothing for other msg */
> > > +	if (r)
> > > +		return 0;
> > > +
> > > +	/* TODO: need to save gfx states */
> > > +	schedule_delayed_work(&adev->virt.flr_work,
> > > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static const struct amdgpu_irq_src_funcs 
> > > +xgpu_vi_mailbox_ack_irq_funcs =
> > > {
> > > +	.set = xgpu_vi_set_mailbox_ack_irq,
> > > +	.process = xgpu_vi_mailbox_ack_irq, };
> > > +
> > > +static const struct amdgpu_irq_src_funcs 
> > > +xgpu_vi_mailbox_rcv_irq_funcs =
> > > {
> > > +	.set = xgpu_vi_set_mailbox_rcv_irq,
> > > +	.process = xgpu_vi_mailbox_rcv_irq, };
> > > +
> > > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> > > +	adev->virt.ack_irq.num_types = 1;
> > > +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> > > +	adev->virt.rcv_irq.num_types = 1;
> > > +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> > > +
> > > +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> > > +	int r;
> > > +
> > > +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> > > +	if (r)
> > > +		return r;
> > > +
> > > +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> > > +	if (r) {
> > > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > > +		return r;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> > > +	int r;
> > > +
> > > +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> > > +	if (r)
> > > +		return r;
> > > +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> > > +	if (r) {
> > > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > > +		return r;
> > > +	}
> > > +
> > > +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> > > xgpu_vi_mailbox_flr_work);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> > > +	cancel_delayed_work_sync(&adev->virt.flr_work);
> > > +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> > > +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> > > +
> > > +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> > > +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> > > +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> > > +	.reset_gpu		= xgpu_vi_request_reset,
> > > +};
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > new file mode 100644
> > > index 0000000..fd6216e
> > > --- /dev/null
> > > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > > @@ -0,0 +1,55 @@
> > > +/*
> > > + * Copyright 2017 Advanced Micro Devices, Inc.
> > > + *
> > > + * Permission is hereby granted, free of charge, to any person 
> > > +obtaining a
> > > + * copy of this software and associated documentation files (the 
> > > +"Software"),
> > > + * to deal in the Software without restriction, including without 
> > > +limitation
> > > + * the rights to use, copy, modify, merge, publish, distribute, 
> > > +sublicense,
> > > + * and/or sell copies of the Software, and to permit persons to 
> > > +whom the
> > > + * Software is furnished to do so, subject to the following conditions:
> > > + *
> > > + * The above copyright notice and this permission notice shall be 
> > > +included in
> > > + * all copies or substantial portions of the Software.
> > > + *
> > > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> > KIND,
> > > +EXPRESS OR
> > > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> > > +MERCHANTABILITY,
> > > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> > NO
> > > EVENT
> > > +SHALL
> > > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> > > +DAMAGES OR
> > > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> > > +OTHERWISE,
> > > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> > > THE USE
> > > +OR
> > > + * OTHER DEALINGS IN THE SOFTWARE.
> > > + */
> > > +
> > > +#ifndef __MXGPU_VI_H__
> > > +#define __MXGPU_VI_H__
> > > +
> > > +#define VI_MAILBOX_TIMEDOUT	150
> > > +#define VI_MAILBOX_RESET_TIME	12
> > > +
> > > +/* VI mailbox messages request */ enum idh_request {
> > > +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> > > +	IDH_REL_GPU_INIT_ACCESS,
> > > +	IDH_REQ_GPU_FINI_ACCESS,
> > > +	IDH_REL_GPU_FINI_ACCESS,
> > > +	IDH_REQ_GPU_RESET_ACCESS
> > > +};
> > > +
> > > +/* VI mailbox messages data */
> > > +enum idh_event {
> > > +	IDH_CLR_MSG_BUF = 0,
> > > +	IDH_READY_TO_ACCESS_GPU,
> > > +	IDH_FLR_NOTIFICATION,
> > > +	IDH_FLR_NOTIFICATION_CMPL,
> > > +	IDH_EVENT_MAX
> > > +};
> > > +
> > > +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> > > +
> > > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); 
> > > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); 
> > > +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int 
> > > +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void 
> > > +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> > > +
> > > +#endif
> > > --
> > > 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 09/11] drm/amdgpu/virt: enable virtual display

[Liu, Monk]

Reviewed-by: Monk Liu

-----Original Message-----
From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf Of Xiangliang Yu
Sent: Wednesday, January 11, 2017 9:18 PM
To: amd-gfx@lists.freedesktop.org
Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>
Subject: [V3 09/11] drm/amdgpu/virt: enable virtual display

Virtual display is default setting for virtualization, enable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 4e7df8e..4bc373e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -25,6 +25,10 @@
 
 void amdgpu_virt_init_setting(struct amdgpu_device *adev)  {
+	/* enable virtual display */
+	adev->mode_info.num_crtc = 1;
+	adev->enable_virtual_display = true;
+
 	mutex_init(&adev->virt.lock);
 }
 
--
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 10/11] drm/amdgpu/virt: disable dc

[Liu, Monk]

My RB as well

-----Original Message-----
From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf Of Xiangliang Yu
Sent: Wednesday, January 11, 2017 9:18 PM
To: amd-gfx@lists.freedesktop.org
Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>
Subject: [V3 10/11] drm/amdgpu/virt: disable dc

Virtualization don't need the dc, disable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 03d28a1..a6d828c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
  */
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)  {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
 }
 
--
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

[-- Attachment #1.1: Type: text/plain, Size: 11614 bytes --]

Xiangliang


please BUG() when register access occured in RUNTIME and IRQ context, e.g.:


if (amdgpu_sriov_runtime(adev)) {


}
  return amdgpu_virt_kiq_wreg(adev, reg, v);


with above addressed, Reviewed-by: Monk Liu <monk.liu@amd.com>

________________________________
发件人: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> 代表 Liu, Monk <Monk.Liu@amd.com>
发送时间: 2017年1月11日 22:54:52
收件人: Christian König; Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

We should BUG and not access register at all if found during RUNTIME && IRQ_context
By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:deathsimple@vodafone.de]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

> do you think that's okay ?
Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:
> Yeah, make sense
>
> I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?
>
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 10:16 PM
> To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang
> <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
> That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.
>
> If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.
>
> Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.
>
> Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.
>
> Regards,
> Christian.
>
> Am 11.01.2017 um 14:51 schrieb Liu, Monk:
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>>
>> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>>
>> And to fully support register reading, we need to implement a new
>> method to use KIQ access register without any schedule() chance, but
>> that's overhead for current stage, and we don't observe registers
>> reading in IRQ context now on 3D apps. (kfd observed such case, and
>> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register
>> dumping, instead of directly reading them via KIQ )
>>
>> BR Monk
>>
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Wednesday, January 11, 2017 9:38 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Liu, Monk <Monk.Liu@amd.com>
>> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>>
>> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>>> For virtualization, it is must for driver to use KIQ to access
>>> registers when it is out of GPU full access mode.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>>> ---
>>>     drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>>     drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>>     5 files changed, 97 insertions(+), 1 deletion(-)
>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 4185b03..0b8e470 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>>      atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>>      amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>>      amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>>> -   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>>> +   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>>
>>>     # add asic specific block
>>>     amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
>>> kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index f82919d..9a2fd3e 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>>     {
>>>      uint32_t ret;
>>>
>>> +   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>>> +           return amdgpu_virt_kiq_rreg(adev, reg);
>>> +
>> Mhm, why is the in_interrupt() necessary here?
>>
>>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>>      else {
>>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>>     {
>>>      trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>>
>>> +   if (amdgpu_sriov_runtime(adev))
>>> +           return amdgpu_virt_kiq_wreg(adev, reg, v);
>>> +
>> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>>
>> Christian.
>>
>>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>              writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>>      else {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> new file mode 100644
>>> index 0000000..7861b6b
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -0,0 +1,82 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to
>>> +whom the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
>>> +EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
>>> +USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "amdgpu_virt.h"
>>> +
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>>> +   mutex_init(&adev->virt.lock);
>>> +}
>>> +
>>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>>> +reg) {
>>> +   signed long r;
>>> +   uint32_t val;
>>> +   struct fence *f;
>>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +   struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +   BUG_ON(!ring->funcs->emit_rreg);
>>> +
>>> +   mutex_lock(&adev->virt.lock);
>>> +   amdgpu_ring_alloc(ring, 32);
>>> +   amdgpu_ring_emit_hdp_flush(ring);
>>> +   amdgpu_ring_emit_rreg(ring, reg);
>>> +   amdgpu_ring_emit_hdp_invalidate(ring);
>>> +   amdgpu_fence_emit(ring, &f);
>>> +   amdgpu_ring_commit(ring);
>>> +   mutex_unlock(&adev->virt.lock);
>>> +
>>> +   r = fence_wait(f, false);
>>> +   if (r)
>>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +   fence_put(f);
>>> +
>>> +   val = adev->wb.wb[adev->virt.reg_val_offs];
>>> +
>>> +   return val;
>>> +}
>>> +
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v) {
>>> +   signed long r;
>>> +   struct fence *f;
>>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +   struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +   BUG_ON(!ring->funcs->emit_wreg);
>>> +
>>> +   mutex_lock(&adev->virt.lock);
>>> +   amdgpu_ring_alloc(ring, 32);
>>> +   amdgpu_ring_emit_hdp_flush(ring);
>>> +   amdgpu_ring_emit_wreg(ring, reg, v);
>>> +   amdgpu_ring_emit_hdp_invalidate(ring);
>>> +   amdgpu_fence_emit(ring, &f);
>>> +   amdgpu_ring_commit(ring);
>>> +   mutex_unlock(&adev->virt.lock);
>>> +
>>> +   r = fence_wait(f, false);
>>> +   if (r)
>>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +   fence_put(f);
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 3d38a9f..2869980 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -33,6 +33,7 @@
>>>     struct amdgpu_virt {
>>>      uint32_t                caps;
>>>      uint32_t                reg_val_offs;
>>> +   struct mutex            lock;
>>>     };
>>>
>>>     #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static
>>> inline bool is_virtual_machine(void)
>>>     #endif
>>>     }
>>>
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
>>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v);
>>> +
>>>     #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>>              (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>>              smc_enabled = true;
>>>
>>> +   if (amdgpu_sriov_vf(adev))
>>> +           amdgpu_virt_init_setting(adev);
>>> +
>>>      adev->rev_id = vi_get_rev_id(adev);
>>>      adev->external_rev_id = 0xFF;
>>>      switch (adev->asic_type) {
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[-- Attachment #1.2: Type: text/html, Size: 19394 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

[-- Attachment #1.1: Type: text/plain, Size: 12214 bytes --]

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

for register read:
if (amdgpu_sriov_runtime(adev)) {
   if (in_interrupt())
        BUG();
   else
       return amdgpu_virt_kiq_rreg(adev, reg, v);
}

and also for register write, with above addressed, Reviewed-by: Monk Liu <monk.liu@amd.com>


________________________________
发件人: Liu, Monk
发送时间: 2017年1月12日 11:19:40
收件人: Christian König; Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers


Xiangliang


please BUG() when register access occured in RUNTIME and IRQ context, e.g.:


if (amdgpu_sriov_runtime(adev)) {


}
  return amdgpu_virt_kiq_wreg(adev, reg, v);


with above addressed, Reviewed-by: Monk Liu <monk.liu@amd.com>

________________________________
发件人: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> 代表 Liu, Monk <Monk.Liu@amd.com>
发送时间: 2017年1月11日 22:54:52
收件人: Christian König; Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

We should BUG and not access register at all if found during RUNTIME && IRQ_context
By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:deathsimple@vodafone.de]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

> do you think that's okay ?
Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:
> Yeah, make sense
>
> I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?
>
> BR Monk
>
> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Wednesday, January 11, 2017 10:16 PM
> To: Liu, Monk <Monk.Liu@amd.com>; Yu, Xiangliang
> <Xiangliang.Yu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
> That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.
>
> If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.
>
> Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.
>
> Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.
>
> Regards,
> Christian.
>
> Am 11.01.2017 um 14:51 schrieb Liu, Monk:
>> Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...
>>
>> If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.
>>
>> And to fully support register reading, we need to implement a new
>> method to use KIQ access register without any schedule() chance, but
>> that's overhead for current stage, and we don't observe registers
>> reading in IRQ context now on 3D apps. (kfd observed such case, and
>> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register
>> dumping, instead of directly reading them via KIQ )
>>
>> BR Monk
>>
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Wednesday, January 11, 2017 9:38 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Liu, Monk <Monk.Liu@amd.com>
>> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>>
>> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
>>> For virtualization, it is must for driver to use KIQ to access
>>> registers when it is out of GPU full access mode.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
>>> ---
>>>     drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>>     drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>>     5 files changed, 97 insertions(+), 1 deletion(-)
>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 4185b03..0b8e470 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>>      atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>>      amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>>      amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>>> -   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>>> +   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>>
>>>     # add asic specific block
>>>     amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
>>> kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index f82919d..9a2fd3e 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>>>     {
>>>      uint32_t ret;
>>>
>>> +   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>>> +           return amdgpu_virt_kiq_rreg(adev, reg);
>>> +
>> Mhm, why is the in_interrupt() necessary here?
>>
>>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>>      else {
>>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>>>     {
>>>      trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>>
>>> +   if (amdgpu_sriov_runtime(adev))
>>> +           return amdgpu_virt_kiq_wreg(adev, reg, v);
>>> +
>> And why it isn't necessary here? What happens when we write a register from an interrupt routine?
>>
>> Christian.
>>
>>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>              writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>>      else {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> new file mode 100644
>>> index 0000000..7861b6b
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -0,0 +1,82 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to
>>> +whom the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
>>> +EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
>>> +USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "amdgpu_virt.h"
>>> +
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>>> +   mutex_init(&adev->virt.lock);
>>> +}
>>> +
>>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>>> +reg) {
>>> +   signed long r;
>>> +   uint32_t val;
>>> +   struct fence *f;
>>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +   struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +   BUG_ON(!ring->funcs->emit_rreg);
>>> +
>>> +   mutex_lock(&adev->virt.lock);
>>> +   amdgpu_ring_alloc(ring, 32);
>>> +   amdgpu_ring_emit_hdp_flush(ring);
>>> +   amdgpu_ring_emit_rreg(ring, reg);
>>> +   amdgpu_ring_emit_hdp_invalidate(ring);
>>> +   amdgpu_fence_emit(ring, &f);
>>> +   amdgpu_ring_commit(ring);
>>> +   mutex_unlock(&adev->virt.lock);
>>> +
>>> +   r = fence_wait(f, false);
>>> +   if (r)
>>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +   fence_put(f);
>>> +
>>> +   val = adev->wb.wb[adev->virt.reg_val_offs];
>>> +
>>> +   return val;
>>> +}
>>> +
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v) {
>>> +   signed long r;
>>> +   struct fence *f;
>>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +   struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +   BUG_ON(!ring->funcs->emit_wreg);
>>> +
>>> +   mutex_lock(&adev->virt.lock);
>>> +   amdgpu_ring_alloc(ring, 32);
>>> +   amdgpu_ring_emit_hdp_flush(ring);
>>> +   amdgpu_ring_emit_wreg(ring, reg, v);
>>> +   amdgpu_ring_emit_hdp_invalidate(ring);
>>> +   amdgpu_fence_emit(ring, &f);
>>> +   amdgpu_ring_commit(ring);
>>> +   mutex_unlock(&adev->virt.lock);
>>> +
>>> +   r = fence_wait(f, false);
>>> +   if (r)
>>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +   fence_put(f);
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 3d38a9f..2869980 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -33,6 +33,7 @@
>>>     struct amdgpu_virt {
>>>      uint32_t                caps;
>>>      uint32_t                reg_val_offs;
>>> +   struct mutex            lock;
>>>     };
>>>
>>>     #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static
>>> inline bool is_virtual_machine(void)
>>>     #endif
>>>     }
>>>
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
>>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v);
>>> +
>>>     #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>>              (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>>>              smc_enabled = true;
>>>
>>> +   if (amdgpu_sriov_vf(adev))
>>> +           amdgpu_virt_init_setting(adev);
>>> +
>>>      adev->rev_id = vi_get_rev_id(adev);
>>>      adev->external_rev_id = 0xFF;
>>>      switch (adev->asic_type) {
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[-- Attachment #1.2: Type: text/html, Size: 21497 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Michel Dänzer]

On 12/01/17 12:21 PM, Liu, Monk wrote:
> 
>    if (in_interrupt())
>         BUG();

Current coding style is

    BUG_ON(in_interrupt());

according to https://kernelnewbies.org/FAQ/BUG .


-- 
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Yu, Xiangliang]

> -----Original Message-----
> From: Michel Dänzer [mailto:michel@daenzer.net]
> Sent: Thursday, January 12, 2017 1:45 PM
> To: Liu, Monk <Monk.Liu@amd.com>; Christian König
> <deathsimple@vodafone.de>; Yu, Xiangliang <Xiangliang.Yu@amd.com>
> Cc: amd-gfx@lists.freedesktop.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
> 
> On 12/01/17 12:21 PM, Liu, Monk wrote:
> >
> >    if (in_interrupt())
> >         BUG();
> 
> Current coding style is
> 
>     BUG_ON(in_interrupt());
> 
> according to https://kernelnewbies.org/FAQ/BUG .

Got it. Thanks!

> 
> 
> --
> Earthling Michel Dänzer               |               http://www.amd.com
> Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

[-- Attachment #1.1: Type: text/plain, Size: 13704 bytes --]

I would go a step further and put the BUG_ON(in_interrupt()) into 
amdgpu_virt_kiq_rreg().

And BTW the problem is even worse, you also can't lock a mutex inside an 
atomic section and we could have plenty of that as well in the driver code.

Regards,
Christian.

Am 12.01.2017 um 04:21 schrieb Liu, Monk:
>
> Xiangliang
>
> please BUG() when register access occured in RUNTIME and IRQ context, 
> e.g.:
>
> for register read:
> if (amdgpu_sriov_runtime(adev)) {
>    if (in_interrupt())
>         BUG();
>    else
> return amdgpu_virt_kiq_rreg(adev, reg, v);
> }
>
> and also for register write, with above addressed, Reviewed-by: Monk 
> Liu <monk.liu-5C7GfCeVMHo@public.gmane.org>
>
> ------------------------------------------------------------------------
> *发件人:* Liu, Monk
> *发送时间:* 2017年1月12日 11:19:40
> *收件人:* Christian König; Yu, Xiangliang; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> *主题:* 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
> Xiangliang
>
>
> please BUG() when register access occured in RUNTIME and IRQ context, 
> e.g.:
>
>
> if (amdgpu_sriov_runtime(adev)) {
>
>
> }
>   return amdgpu_virt_kiq_wreg(adev, reg, v);
>
>
> with above addressed, Reviewed-by: Monk Liu <monk.liu-5C7GfCeVMHo@public.gmane.org>
>
> ------------------------------------------------------------------------
> *发件人:* amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> 代表 Liu, Monk 
> <Monk.Liu-5C7GfCeVMHo@public.gmane.org>
> *发送时间:* 2017年1月11日 22:54:52
> *收件人:* Christian König; Yu, Xiangliang; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> *主题:* RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
> We should BUG and not access register at all if found during RUNTIME 
> && IRQ_context
> By far.
>
> BR Monk
> -----Original Message-----
> From: Christian König [mailto:deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org]
> Sent: Wednesday, January 11, 2017 10:44 PM
> To: Liu, Monk <Monk.Liu-5C7GfCeVMHo@public.gmane.org>; Yu, Xiangliang 
> <Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org>; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
>
> > do you think that's okay ?
> Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup 
> when a client writes to some registers from interrupt context.
>
> Christian.
>
> Am 11.01.2017 um 15:38 schrieb Liu, Monk:
> > Yeah, make sense
> >
> > I think before we have a full version of KIQ reg accessing without 
> context switch, we should totally disable KIQ reg read/write in IRQ 
> context, do you think that's okay ?
> >
> > BR Monk
> >
> > -----Original Message-----
> > From: Christian König [mailto:deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org]
> > Sent: Wednesday, January 11, 2017 10:16 PM
> > To: Liu, Monk <Monk.Liu-5C7GfCeVMHo@public.gmane.org>; Yu, Xiangliang
> > <Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org>; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> > Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
> >
> >> Because if we are in interrupt , we are forbid to do schedule, and 
> use kiq to read register will invoke fence_wait() ...
> > That won't work. Locking a mutext like it is done in the write path 
> can cause scheduling as well.
> >
> > If we need to push writes through the KIQ in interrupt context we 
> need to change the code to use a spinlock with disabled interrupts 
> instead of a mutex.
> >
> > Otherwise the whole thing can easily deadlock when an interrupt 
> happens to come while the lock is taken.
> >
> > Please also test the patchset with lockdep enabled, that should 
> yield a bunch of error messages when you try to do something like this.
> >
> > Regards,
> > Christian.
> >
> > Am 11.01.2017 um 14:51 schrieb Liu, Monk:
> >> Because if we are in interrupt , we are forbid to do schedule, and 
> use kiq to read register will invoke fence_wait() ...
> >>
> >> If you think that's odds, we can use BUG_ON() to stop driver 
> running if we need read registers in SRIOV case.
> >>
> >> And to fully support register reading, we need to implement a new
> >> method to use KIQ access register without any schedule() chance, but
> >> that's overhead for current stage, and we don't observe registers
> >> reading in IRQ context now on 3D apps. (kfd observed such case, and
> >> @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register
> >> dumping, instead of directly reading them via KIQ )
> >>
> >> BR Monk
> >>
> >> -----Original Message-----
> >> From: Christian König [mailto:deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org]
> >> Sent: Wednesday, January 11, 2017 9:38 PM
> >> To: Yu, Xiangliang <Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org>;
> >> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> >> Cc: Liu, Monk <Monk.Liu-5C7GfCeVMHo@public.gmane.org>
> >> Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
> >>
> >> Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:
> >>> For virtualization, it is must for driver to use KIQ to access
> >>> registers when it is out of GPU full access mode.
> >>>
> >>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org>
> >>> Signed-off-by: Monk Liu <Monk.Liu-5C7GfCeVMHo@public.gmane.org>
> >>> ---
> >>> drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
> >>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
> >>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 
> ++++++++++++++++++++++++++++++
> >>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
> >>> drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
> >>>     5 files changed, 97 insertions(+), 1 deletion(-)
> >>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> index 4185b03..0b8e470 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
> >>>      atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
> >>>      amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
> >>>      amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> >>> -   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> >>> +   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
> >>>
> >>>     # add asic specific block
> >>>     amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
> >>> kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> index f82919d..9a2fd3e 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device 
> *adev, uint32_t reg,
> >>>     {
> >>>      uint32_t ret;
> >>>
> >>> +   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> >>> +           return amdgpu_virt_kiq_rreg(adev, reg);
> >>> +
> >> Mhm, why is the in_interrupt() necessary here?
> >>
> >>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >>>              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
> >>>      else {
> >>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device 
> *adev, uint32_t reg, uint32_t v,
> >>>     {
> >>> trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
> >>>
> >>> +   if (amdgpu_sriov_runtime(adev))
> >>> +           return amdgpu_virt_kiq_wreg(adev, reg, v);
> >>> +
> >> And why it isn't necessary here? What happens when we write a 
> register from an interrupt routine?
> >>
> >> Christian.
> >>
> >>>      if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >>>              writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
> >>>      else {
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> new file mode 100644
> >>> index 0000000..7861b6b
> >>> --- /dev/null
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> @@ -0,0 +1,82 @@
> >>> +/*
> >>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>> + *
> >>> + * Permission is hereby granted, free of charge, to any person
> >>> +obtaining a
> >>> + * copy of this software and associated documentation files (the
> >>> +"Software"),
> >>> + * to deal in the Software without restriction, including without
> >>> +limitation
> >>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>> +sublicense,
> >>> + * and/or sell copies of the Software, and to permit persons to
> >>> +whom the
> >>> + * Software is furnished to do so, subject to the following 
> conditions:
> >>> + *
> >>> + * The above copyright notice and this permission notice shall be
> >>> +included in
> >>> + * all copies or substantial portions of the Software.
> >>> + *
> >>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> >>> +EXPRESS OR
> >>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>> +MERCHANTABILITY,
> >>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> >>> +EVENT SHALL
> >>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> >>> +DAMAGES OR
> >>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> >>> +OTHERWISE,
> >>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
> >>> +USE OR
> >>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>> + */
> >>> +
> >>> +#include "amdgpu.h"
> >>> +#include "amdgpu_virt.h"
> >>> +
> >>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> >>> +   mutex_init(&adev->virt.lock);
> >>> +}
> >>> +
> >>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> >>> +reg) {
> >>> +   signed long r;
> >>> +   uint32_t val;
> >>> +   struct fence *f;
> >>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> >>> +   struct amdgpu_ring *ring = &kiq->ring;
> >>> +
> >>> +   BUG_ON(!ring->funcs->emit_rreg);
> >>> +
> >>> +   mutex_lock(&adev->virt.lock);
> >>> +   amdgpu_ring_alloc(ring, 32);
> >>> +   amdgpu_ring_emit_hdp_flush(ring);
> >>> +   amdgpu_ring_emit_rreg(ring, reg);
> >>> +   amdgpu_ring_emit_hdp_invalidate(ring);
> >>> +   amdgpu_fence_emit(ring, &f);
> >>> +   amdgpu_ring_commit(ring);
> >>> +   mutex_unlock(&adev->virt.lock);
> >>> +
> >>> +   r = fence_wait(f, false);
> >>> +   if (r)
> >>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >>> +   fence_put(f);
> >>> +
> >>> +   val = adev->wb.wb[adev->virt.reg_val_offs];
> >>> +
> >>> +   return val;
> >>> +}
> >>> +
> >>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> >>> +uint32_t v) {
> >>> +   signed long r;
> >>> +   struct fence *f;
> >>> +   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> >>> +   struct amdgpu_ring *ring = &kiq->ring;
> >>> +
> >>> +   BUG_ON(!ring->funcs->emit_wreg);
> >>> +
> >>> +   mutex_lock(&adev->virt.lock);
> >>> +   amdgpu_ring_alloc(ring, 32);
> >>> +   amdgpu_ring_emit_hdp_flush(ring);
> >>> +   amdgpu_ring_emit_wreg(ring, reg, v);
> >>> +   amdgpu_ring_emit_hdp_invalidate(ring);
> >>> +   amdgpu_fence_emit(ring, &f);
> >>> +   amdgpu_ring_commit(ring);
> >>> +   mutex_unlock(&adev->virt.lock);
> >>> +
> >>> +   r = fence_wait(f, false);
> >>> +   if (r)
> >>> +           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >>> +   fence_put(f);
> >>> +}
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> index 3d38a9f..2869980 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> @@ -33,6 +33,7 @@
> >>>     struct amdgpu_virt {
> >>>      uint32_t                caps;
> >>>      uint32_t                reg_val_offs;
> >>> +   struct mutex            lock;
> >>>     };
> >>>
> >>>     #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static
> >>> inline bool is_virtual_machine(void)
> >>>     #endif
> >>>     }
> >>>
> >>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
> >>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
> >>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> >>> +uint32_t v);
> >>> +
> >>>     #endif
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
> >>>              (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
> >>>              smc_enabled = true;
> >>>
> >>> +   if (amdgpu_sriov_vf(adev))
> >>> +           amdgpu_virt_init_setting(adev);
> >>> +
> >>>      adev->rev_id = vi_get_rev_id(adev);
> >>>      adev->external_rev_id = 0xFF;
> >>>      switch (adev->asic_type) {
> >> _______________________________________________
> >> amd-gfx mailing list
> >> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> >> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> >
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx



[-- Attachment #1.2: Type: text/html, Size: 29153 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx