[V2 00/11] Add support AMD GPU virtualization soultion

[V2 00/11] Add support AMD GPU virtualization soultion

[Xiangliang Yu]

This patch series will enable AMD GPU virtualization and wires it
into amdgpu modules.

Changes in V2:
1. define new high level interface for non-asic specific;
2. add asic specific file to support specific asic;
3. not include KIQ patch as it has been merged into kernel;
4. not include CSA patch as monk will submit it;

Xiangliang Yu (11):
  drm/amdgpu/ring: add two interfaces to support r/w registers with kiq
  drm/amdgpu/gfx8: implement emit_rreg/wreg function
  drm/amdgpu/virt: add runtime flag
  drm/amdgpu/virt: use kiq to access registers
  drm/amdgpu/virt: add high level interfaces for virt
  drm/amdgpu/virt: implement VI virt operation interfaces
  drm/amdgpu: request/release full gpu access if device is vf
  drm/amdgpu/vi: add support virtualization
  drm/amdgpu/virt: enable virtual display
  drm/amdgpu/virt: disable dc
  drm/amdgpu: do not reset gpu for virtualization

 drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592 +++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
 11 files changed, 926 insertions(+), 4 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 01/11] drm/amdgpu/ring: add two interfaces to support r/w registers with kiq

[Xiangliang Yu]

During virtual runtime, need to send command to kiq ring to
read/write GPU registers. Add two interface to support the two
actions.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h      | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 13c30ab..13d87d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1646,6 +1646,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
 #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
+#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
+#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 92bc89b..c813cbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -135,6 +135,8 @@ struct amdgpu_ring_funcs {
 	void (*end_use)(struct amdgpu_ring *ring);
 	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
 	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
+	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 };
 
 struct amdgpu_ring {
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg function

[Xiangliang Yu]

Implement emit_rreg/wreg function for kiq ring.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  3 ++-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 0d821d9..a6524bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -30,7 +30,8 @@
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t caps;
+	uint32_t		caps;
+	uint32_t		val_offs;
 };
 
 #define amdgpu_sriov_enabled(adev) \
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index b487b97..e619f49 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1373,6 +1373,12 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 {
 	int r = 0;
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_wb_get(adev, &adev->virt.val_offs);
+		if (r)
+			return r;
+	}
+
 	ring->adev = NULL;
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
@@ -1399,6 +1405,9 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
 				   struct amdgpu_irq_src *irq)
 {
+	if (amdgpu_sriov_vf(ring->adev))
+		amdgpu_wb_free(ring->adev, ring->adev->virt.val_offs);
+
 	amdgpu_ring_fini(ring);
 	irq->data = NULL;
 }
@@ -6681,6 +6690,32 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 	amdgpu_ring_write(ring, 0);
 }
 
+static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+	amdgpu_ring_write(ring, 0 |	/* src: register*/
+				(5 << 8) |	/* dst: memory */
+				(1 << 20));	/* write confirm */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+				adev->virt.val_offs * 4));
+	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+				adev->virt.val_offs * 4));
+}
+
+static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+				  uint32_t val)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, val);
+}
+
 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 						 enum amdgpu_interrupt_state state)
 {
@@ -6995,6 +7030,8 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_rreg = gfx_v8_0_ring_emit_rreg,
+	.emit_wreg = gfx_v8_0_ring_emit_wreg,
 };
 
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 03/11] drm/amdgpu/virt: add runtime flag

[Xiangliang Yu]

Add new flag to define gpu runtime that is out of full gpu access.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index a6524bb..79619b6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -28,6 +28,7 @@
 #define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled on this GPU */
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
+#define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
@@ -43,6 +44,9 @@ struct amdgpu_virt {
 #define amdgpu_sriov_bios(adev) \
 ((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
 
+#define amdgpu_sriov_runtime(adev) \
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME)
+
 #define amdgpu_passthrough(adev) \
 ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Xiangliang Yu]

For virtualization, it is must for driver to use KIQ to access
registers when it is out of GPU full access mode.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 ++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
 5 files changed, 101 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 4185b03..0b8e470 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 {
 	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 {
 	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
new file mode 100644
index 0000000..6520a4e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev)
+{
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
+{
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	fence_get(f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	fence_put(f);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	fence_get(f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	fence_put(f);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 79619b6..24f0590 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -33,6 +33,7 @@
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		val_offs;
+	struct mutex		lock;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
 #endif
 }
 
+void amdgpu_virt_init_setting(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 7350a8f..dc0d4fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
 	switch (adev->asic_type) {
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Xiangliang Yu]

Add high level interfaces that is not relate to specific asic. So
asic files just need to implement the interfaces to support
virtualization.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
 2 files changed, 72 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 6520a4e..f32a789 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 24f0590..3f8fc0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -29,11 +29,23 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations
+ */
+struct amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev);
+};
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		val_offs;
 	struct mutex		lock;
+
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +75,8 @@ static inline bool is_virtual_machine(void)
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Xiangliang Yu]

VI has asic specific virt support, which including mailbox and
golden registers init.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 651 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 0b8e470..388e7b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 3f8fc0f..611676b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -45,6 +45,9 @@ struct amdgpu_virt {
 	uint32_t		val_offs;
 	struct mutex		lock;
 
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
new file mode 100644
index 0000000..7d44663
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Xiangliang.Yu@amd.com
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
+};
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+};
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_settings_a10,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_settings_a10));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_golden_common_all));
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_settings_a11,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_settings_a11));
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_golden_common_all,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_tonga_golden_common_all));
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs
+ */
+static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool val)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+				 TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+				 MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true);
+}
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev)
+{
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event event)
+{
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev)
+{
+	return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
+}
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
+	return xgpu_vi_send_access_requests(adev, event);
+}
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state)
+{
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
+{
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */
+}
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state)
+{
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs = {
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs;
+}
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work, xgpu_vi_mailbox_flr_work);
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev)
+{
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+}
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
new file mode 100644
index 0000000..fd6216e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev);
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 07/11] drm/amdgpu: request/release full gpu access if device is vf

[Xiangliang Yu]

For gpu vf device, first need to request full gpu access before
accessing gpu registers, and release full gpu access after the
access is done.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 9 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 9a2fd3e..03d28a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1330,6 +1330,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_request_full_gpu(adev, true);
+		if (r)
+			return r;
+	}
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
 			DRM_ERROR("disabled ip block: %d\n", i);
@@ -1523,6 +1529,9 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.late_initialized = false;
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, false);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 47bc8e1..eb228c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -60,6 +60,9 @@ int amdgpu_driver_unload_kms(struct drm_device *dev)
 	if (adev->rmmio == NULL)
 		goto done_free;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
 	if (amdgpu_device_is_px(dev)) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
@@ -139,6 +142,9 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 		pm_runtime_put_autosuspend(dev->dev);
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, true);
+
 out:
 	if (r) {
 		/* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 08/11] drm/amdgpu/vi: add support virtualization

[Xiangliang Yu]

Call VI virtualization functions if device is Vf.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/vi.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index dc0d4fa..34aa603 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -77,6 +77,7 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_powerplay.h"
 #include "dce_virtual.h"
+#include "mxgpu_vi.h"
 
 /*
  * Indirect registers accessor
@@ -273,6 +274,12 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 	mutex_lock(&adev->grbm_idx_mutex);
 
+	if (amdgpu_sriov_vf(adev)) {
+		xgpu_vi_init_golden_registers(adev);
+		mutex_unlock(&adev->grbm_idx_mutex);
+		return;
+	}
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		amdgpu_program_register_sequence(adev,
@@ -892,8 +899,10 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
-	if (amdgpu_sriov_vf(adev))
+	if (amdgpu_sriov_vf(adev)) {
 		amdgpu_virt_init_setting(adev);
+		xgpu_vi_mailbox_set_irq_funcs(adev);
+	}
 
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
@@ -1059,8 +1068,23 @@ static int vi_common_early_init(void *handle)
 	return 0;
 }
 
+static int vi_common_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_get_irq(adev);
+
+	return 0;
+}
+
 static int vi_common_sw_init(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_add_irq_id(adev);
+
 	return 0;
 }
 
@@ -1092,6 +1116,9 @@ static int vi_common_hw_fini(void *handle)
 	/* enable the doorbell aperture */
 	vi_enable_doorbell_aperture(adev, false);
 
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_put_irq(adev);
+
 	return 0;
 }
 
@@ -1376,7 +1403,7 @@ static int vi_common_set_powergating_state(void *handle,
 static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.name = "vi_common",
 	.early_init = vi_common_early_init,
-	.late_init = NULL,
+	.late_init = vi_common_late_init,
 	.sw_init = vi_common_sw_init,
 	.sw_fini = vi_common_sw_fini,
 	.hw_init = vi_common_hw_init,
@@ -1404,6 +1431,9 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 	/* in early init stage, vbios code won't work */
 	vi_detect_hw_virtualization(adev);
 
+	if (amdgpu_sriov_vf(adev))
+		adev->virt.ops = &xgpu_vi_virt_ops;
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		/* topaz has no DCE, UVD, VCE */
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 09/11] drm/amdgpu/virt: enable virtual display

[Xiangliang Yu]

Virtual display is default setting for virtualization, enable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index f32a789..5baca9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -25,6 +25,9 @@
 
 void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 {
+	/* enable virtual display */
+	adev->mode_info.num_crtc = 1;
+
 	mutex_init(&adev->virt.lock);
 }
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 10/11] drm/amdgpu/virt: disable dc

[Xiangliang Yu]

Virtualization don't need the dc, disable it.

Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 03d28a1..a6d828c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
  */
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
 }
 
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[V2 11/11] drm/amdgpu: do not reset gpu for virtualization

[Xiangliang Yu]

Current job timeout setting is not fit for virtualization, so it
will cause job timeout sometimes. Add workaround for this when
timeout happen.

Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index a6d828c..5705fbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 	int resched;
 	bool need_full_reset;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	if (!amdgpu_check_soft_reset(adev)) {
 		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
 		return 0;
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization

[Liu, Monk]

[-- Attachment #1.1: Type: text/plain, Size: 1674 bytes --]

Xiangliang,


the comments of yours is a little inappropriate, please change the comments to something like:

amdgpu_gpu_reset() only works on bare-metal case and for SRIOV many inside it need re-work to adapt to VF device

so currently just don't do it.


________________________________
发件人: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> 代表 Xiangliang Yu <Xiangliang.Yu@amd.com>
发送时间: 2017年1月10日 18:00:50
收件人: amd-gfx@lists.freedesktop.org
抄送: Yu, Xiangliang; Liu, Monk
主题: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization

Current job timeout setting is not fit for virtualization, so it
will cause job timeout sometimes. Add workaround for this when
timeout happen.

Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index a6d828c..5705fbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
         int resched;
         bool need_full_reset;

+       if (amdgpu_sriov_vf(adev))
+               return 0;
+
         if (!amdgpu_check_soft_reset(adev)) {
                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
                 return 0;
--
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[-- Attachment #1.2: Type: text/html, Size: 3467 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 101 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>   	uint32_t ret;
>   
> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +		return amdgpu_virt_kiq_rreg(adev, reg);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>   
> +	if (amdgpu_sriov_runtime(adev))
> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..6520a4e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,86 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev)
> +{
> +	mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
> +{
> +	signed long r;
> +	uint32_t val;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_rreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +
> +	val = adev->wb.wb[adev->virt.val_offs];
> +
> +	return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
> +{
> +	signed long r;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_wreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_wreg(ring, reg, v);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 79619b6..24f0590 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		val_offs;
> +	struct mutex		lock;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \
> @@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
>   #endif
>   }
>   
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev);
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
> index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>   		smc_enabled = true;
>   
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_setting(adev);
> +
>   	adev->rev_id = vi_get_rev_id(adev);
>   	adev->external_rev_id = 0xFF;
>   	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Christian König]

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> Add high level interfaces that is not relate to specific asic. So
> asic files just need to implement the interfaces to support
> virtualization.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
>   2 files changed, 72 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 6520a4e..f32a789 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
>   		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>   	fence_put(f);
>   }
> +
> +/**
> + * amdgpu_virt_request_full_gpu() - request full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When start to init/fini driver, first need to request full gpu access.
> + * Return: Zero if request success, otherwise will return error.
> + */
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->req_full_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);

I would be conservative here and request full GPU access first and then 
clear AMDGPU_SRIOV_CAPS_RUNTIME.

Just in case the function is called concurrently with another thread 
checking the caps.

On the other hand req_full_gpu() could need the flag to handle register 
reads/writes correctly, but in this case I would question if we 
shouldn't add special macros for this.

Christian.



> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_release_full_gpu() - release full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When finishing driver init/fini, need to release full gpu access.
> + * Return: Zero if release success, otherwise will returen error.
> + */
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +	int r;
> +
> +	if (virt->ops && virt->ops->rel_full_gpu) {
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return r;
> +	}
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_reset_gpu() - reset gpu
> + * @amdgpu:	amdgpu device.
> + * Send reset command to GPU hypervisor to reset GPU that VM is using
> + * Return: Zero if reset success, otherwise will return error.
> + */
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->reset_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> +
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 24f0590..3f8fc0f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -29,11 +29,23 @@
>   #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
>   #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
>   #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
> +
> +/**
> + * struct amdgpu_virt_ops - amdgpu device virt operations
> + */
> +struct amdgpu_virt_ops {
> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*reset_gpu)(struct amdgpu_device *adev);
> +};
> +
>   /* GPU virtualization */
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		val_offs;
>   	struct mutex		lock;
> +
> +	const struct amdgpu_virt_ops	*ops;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \
> @@ -63,5 +75,8 @@ static inline bool is_virtual_machine(void)
>   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
>   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
>   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
>   
>   #endif


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Christian König]

Unrelated to the patch, but what editor and settings do you use?

See this code for example
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> +				 TRN_MSG_VALID, val ? 1 : 0);

The indentation seems to be incorrect.

Not much of a problem, but if we add this like it is anybody else 
editing the file could mess up the indentation when the editor does that 
differently.

Regards,
Christian.

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> VI has asic specific virt support, which including mailbox and
> golden registers init.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
>   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>   4 files changed, 651 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 0b8e470..388e7b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
>   amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
>   
>   amdgpu-y += \
> -	vi.o
> +	vi.o mxgpu_vi.o
>   
>   # add GMC block
>   amdgpu-y += \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 3f8fc0f..611676b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -45,6 +45,9 @@ struct amdgpu_virt {
>   	uint32_t		val_offs;
>   	struct mutex		lock;
>   
> +	struct amdgpu_irq_src		ack_irq;
> +	struct amdgpu_irq_src		rcv_irq;
> +	struct delayed_work		flr_work;
>   	const struct amdgpu_virt_ops	*ops;
>   };
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> new file mode 100644
> index 0000000..7d44663
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> @@ -0,0 +1,592 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors: Xiangliang.Yu@amd.com
> + */
> +
> +#include "amdgpu.h"
> +#include "vi.h"
> +#include "bif/bif_5_0_d.h"
> +#include "bif/bif_5_0_sh_mask.h"
> +#include "vid.h"
> +#include "gca/gfx_8_0_d.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "gmc_v8_0.h"
> +#include "gfx_v8_0.h"
> +#include "sdma_v3_0.h"
> +#include "tonga_ih.h"
> +#include "gmc/gmc_8_2_d.h"
> +#include "gmc/gmc_8_2_sh_mask.h"
> +#include "oss/oss_3_0_d.h"
> +#include "oss/oss_3_0_sh_mask.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "dce/dce_10_0_d.h"
> +#include "dce/dce_10_0_sh_mask.h"
> +#include "smu/smu_7_1_3_d.h"
> +#include "mxgpu_vi.h"
> +
> +/* VI golden setting */
> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +};
> +
> +static const u32 xgpu_fiji_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
> +};
> +
> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +};
> +
> +static const u32 xgpu_tonga_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +};
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
> +{
> +	switch (adev->asic_type) {
> +	case CHIP_FIJI:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_golden_settings_a10,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_golden_settings_a10));
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_golden_common_all));
> +		break;
> +	case CHIP_TONGA:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_tonga_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_golden_settings_a11,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_tonga_golden_settings_a11));
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_tonga_golden_common_all));
> +		break;
> +	default:
> +		BUG_ON("Doesn't support chip type.\n");
> +		break;
> +	}
> +}
> +
> +/*
> + * Mailbox communication between GPU hypervisor and VFs
> + */
> +static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool val)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> +				 TRN_MSG_VALID, val ? 1 : 0);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> +				      enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> +				 MSGBUF_DATA, event);
> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> +
> +	xgpu_vi_mailbox_set_valid(adev, true);
> +}
> +
> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> +				   enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> +	if (reg != event)
> +		return -ENOENT;
> +
> +	/* send ack to PF */
> +	xgpu_vi_mailbox_send_ack(adev);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev)
> +{
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	while (!(reg & mask)) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		reg = RREG32(mmMAILBOX_CONTROL);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event event)
> +{
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +
> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	while (r) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> +					enum idh_request request)
> +{
> +	int r;
> +
> +	xgpu_vi_mailbox_trans_msg(adev, request);
> +
> +	/* start to poll ack */
> +	r = xgpu_vi_poll_ack(adev);
> +	if (r)
> +		return r;
> +
> +	xgpu_vi_mailbox_set_valid(adev, false);
> +
> +	/* start to check msg if request is idh_req_gpu_init_access */
> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> +		if (r)
> +			return r;
> +	}
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_request_reset(struct amdgpu_device *adev)
> +{
> +	return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
> +}
> +
> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +
> +	event = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
> +	return xgpu_vi_send_access_requests(adev, event);
> +}
> +
> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +	int r = 0;
> +
> +	event = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
> +	r = xgpu_vi_send_access_requests(adev, event);
> +
> +	return r;
> +}
> +
> +/* add support mailbox interrupts */
> +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	DRM_DEBUG("get ack intr and do nothing.\n");
> +	return 0;
> +}
> +
> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state)
> +{
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
> +{
> +	struct amdgpu_virt *virt = container_of(work,
> +					struct amdgpu_virt, flr_work.work);
> +	struct amdgpu_device *adev = container_of(virt,
> +					struct amdgpu_device, virt);
> +	int r = 0;
> +
> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> +	if (r)
> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> +
> +	/* TODO: need to restore gfx states */
> +}
> +
> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state)
> +{
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs = {
> +	.set = xgpu_vi_set_mailbox_ack_irq,
> +	.process = xgpu_vi_mailbox_ack_irq,
> +};
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs = {
> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> +	.process = xgpu_vi_mailbox_rcv_irq,
> +};
> +
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
> +{
> +	adev->virt.ack_irq.num_types = 1;
> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> +	adev->virt.rcv_irq.num_types = 1;
> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs;
> +}
> +
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev)
> +{
> +	int r;
> +
> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> +	if (r)
> +		return r;
> +
> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	return 0;
> +}
> +
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev)
> +{
> +	int r;
> +
> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> +	if (r)
> +		return r;
> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	INIT_DELAYED_WORK(&adev->virt.flr_work, xgpu_vi_mailbox_flr_work);
> +
> +	return 0;
> +}
> +
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev)
> +{
> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +}
> +
> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> +	.reset_gpu		= xgpu_vi_request_reset,
> +};
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> new file mode 100644
> index 0000000..fd6216e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> @@ -0,0 +1,55 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef __MXGPU_VI_H__
> +#define __MXGPU_VI_H__
> +
> +#define VI_MAILBOX_TIMEDOUT	150
> +#define VI_MAILBOX_RESET_TIME	12
> +
> +/* VI mailbox messages request */
> +enum idh_request {
> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> +	IDH_REL_GPU_INIT_ACCESS,
> +	IDH_REQ_GPU_FINI_ACCESS,
> +	IDH_REL_GPU_FINI_ACCESS,
> +	IDH_REQ_GPU_RESET_ACCESS
> +};
> +
> +/* VI mailbox messages data */
> +enum idh_event {
> +	IDH_CLR_MSG_BUF = 0,
> +	IDH_READY_TO_ACCESS_GPU,
> +	IDH_FLR_NOTIFICATION,
> +	IDH_FLR_NOTIFICATION_CMPL,
> +	IDH_EVENT_MAX
> +};
> +
> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev);
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev);
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> +
> +#endif


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization

[Christian König]

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> Current job timeout setting is not fit for virtualization, so it
> will cause job timeout sometimes. Add workaround for this when
> timeout happen.
>
> Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
>   1 file changed, 3 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index a6d828c..5705fbd 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
>   	int resched;
>   	bool need_full_reset;
>   
> +	if (amdgpu_sriov_vf(adev))

At bare minimum we should print a warning here and increase or 
completely disable the job timeout when virtualization is enabled.

Christian.

> +		return 0;
> +
>   	if (!amdgpu_check_soft_reset(adev)) {
>   		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
>   		return 0;


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:16 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
> interfaces
> 
> Unrelated to the patch, but what editor and settings do you use?

Vim

> 
> See this code for example
> > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> > +				 TRN_MSG_VALID, val ? 1 : 0);
> 
> The indentation seems to be incorrect.

In my platform, there is no problem. 

> 
> Not much of a problem, but if we add this like it is anybody else editing the
> file could mess up the indentation when the editor does that differently.
> 
> Regards,
> Christian.
> 
> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > VI has asic specific virt support, which including mailbox and golden
> > registers init.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
> >   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> +++++++++++++++++++++++++++++++
> >   drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> >   4 files changed, 651 insertions(+), 1 deletion(-)
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > index 0b8e470..388e7b0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> cik_ih.o kv_smc.o kv_dpm.o \
> >   amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o
> si_ih.o
> > si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> >
> >   amdgpu-y += \
> > -	vi.o
> > +	vi.o mxgpu_vi.o
> >
> >   # add GMC block
> >   amdgpu-y += \
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 3f8fc0f..611676b 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -45,6 +45,9 @@ struct amdgpu_virt {
> >   	uint32_t		val_offs;
> >   	struct mutex		lock;
> >
> > +	struct amdgpu_irq_src		ack_irq;
> > +	struct amdgpu_irq_src		rcv_irq;
> > +	struct delayed_work		flr_work;
> >   	const struct amdgpu_virt_ops	*ops;
> >   };
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > new file mode 100644
> > index 0000000..7d44663
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> > @@ -0,0 +1,592 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + *
> > + * Authors: Xiangliang.Yu@amd.com
> > + */
> > +
> > +#include "amdgpu.h"
> > +#include "vi.h"
> > +#include "bif/bif_5_0_d.h"
> > +#include "bif/bif_5_0_sh_mask.h"
> > +#include "vid.h"
> > +#include "gca/gfx_8_0_d.h"
> > +#include "gca/gfx_8_0_sh_mask.h"
> > +#include "gmc_v8_0.h"
> > +#include "gfx_v8_0.h"
> > +#include "sdma_v3_0.h"
> > +#include "tonga_ih.h"
> > +#include "gmc/gmc_8_2_d.h"
> > +#include "gmc/gmc_8_2_sh_mask.h"
> > +#include "oss/oss_3_0_d.h"
> > +#include "oss/oss_3_0_sh_mask.h"
> > +#include "gca/gfx_8_0_sh_mask.h"
> > +#include "dce/dce_10_0_d.h"
> > +#include "dce/dce_10_0_sh_mask.h"
> > +#include "smu/smu_7_1_3_d.h"
> > +#include "mxgpu_vi.h"
> > +
> > +/* VI golden setting */
> > +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> > +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> > +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> > +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> > +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> > +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> > +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> > +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> > +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> > +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> > +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> > +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> > +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> > +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> > +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> > +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> > +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> > +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> > +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> > +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> > +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> > +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> > +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> > +
> > +static const u32 xgpu_fiji_golden_settings_a10[] = {
> > +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > +
> > +static const u32 xgpu_fiji_golden_common_all[] = {
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> > +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> > +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> > +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> > +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> > +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> > +
> > +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> > +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> > +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> > +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> > +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> > +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> > +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> > +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> > +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> > +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> > +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> > +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> > +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> > +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> > +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> > +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> > +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> > +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> > +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> > +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> > +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> > +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> > +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> > +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> > +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> > +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> > +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> > +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> > +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> > +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> > +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> > +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> > +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> > +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> > +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> > +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> > +};
> > +
> > +static const u32 xgpu_tonga_golden_settings_a11[] = {
> > +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> > +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> > +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> > +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> > +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> > +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> > +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> > +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> > +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> > +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> > +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> > +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> > +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> > +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> > +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> > +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> > +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> > +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> > +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> > +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> > +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> > +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> > +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> > +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> > +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> > +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> > +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> > +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> > +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> > +
> > +static const u32 xgpu_tonga_golden_common_all[] = {
> > +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> > +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> > +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> > +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> > +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> > +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> > +
> > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> > +	switch (adev->asic_type) {
> > +	case CHIP_FIJI:
> > +		amdgpu_program_register_sequence(adev,
> > +						 xgpu_fiji_mgcg_cgcg_init,
> > +						 (const u32)ARRAY_SIZE(
> > +						 xgpu_fiji_mgcg_cgcg_init));
> > +		amdgpu_program_register_sequence(adev,
> > +
> xgpu_fiji_golden_settings_a10,
> > +						 (const u32)ARRAY_SIZE(
> > +
> xgpu_fiji_golden_settings_a10));
> > +		amdgpu_program_register_sequence(adev,
> > +
> xgpu_fiji_golden_common_all,
> > +						 (const u32)ARRAY_SIZE(
> > +
> xgpu_fiji_golden_common_all));
> > +		break;
> > +	case CHIP_TONGA:
> > +		amdgpu_program_register_sequence(adev,
> > +						 xgpu_tonga_mgcg_cgcg_init,
> > +						 (const u32)ARRAY_SIZE(
> > +
> xgpu_tonga_mgcg_cgcg_init));
> > +		amdgpu_program_register_sequence(adev,
> > +
> xgpu_tonga_golden_settings_a11,
> > +						 (const u32)ARRAY_SIZE(
> > +
> xgpu_tonga_golden_settings_a11));
> > +		amdgpu_program_register_sequence(adev,
> > +
> xgpu_tonga_golden_common_all,
> > +						 (const u32)ARRAY_SIZE(
> > +
> xgpu_tonga_golden_common_all));
> > +		break;
> > +	default:
> > +		BUG_ON("Doesn't support chip type.\n");
> > +		break;
> > +	}
> > +}
> > +
> > +/*
> > + * Mailbox communication between GPU hypervisor and VFs  */ static
> > +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> > +	WREG32(mmMAILBOX_CONTROL, reg);
> > +}
> > +
> > +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
> > +bool val) {
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> > +				 TRN_MSG_VALID, val ? 1 : 0);
> > +	WREG32(mmMAILBOX_CONTROL, reg);
> > +}
> > +
> > +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> > +				      enum idh_event event)
> > +{
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> > +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> > +				 MSGBUF_DATA, event);
> > +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> > +
> > +	xgpu_vi_mailbox_set_valid(adev, true); }
> > +
> > +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> > +				   enum idh_event event)
> > +{
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> > +	if (reg != event)
> > +		return -ENOENT;
> > +
> > +	/* send ack to PF */
> > +	xgpu_vi_mailbox_send_ack(adev);
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> > +	u32 reg;
> > +
> > +	reg = RREG32(mmMAILBOX_CONTROL);
> > +	while (!(reg & mask)) {
> > +		if (timeout <= 0) {
> > +			pr_err("Doesn't get ack from pf.\n");
> > +			r = -ETIME;
> > +			break;
> > +		}
> > +		msleep(1);
> > +		timeout -= 1;
> > +
> > +		reg = RREG32(mmMAILBOX_CONTROL);
> > +	}
> > +
> > +	return r;
> > +}
> > +
> > +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
> > +idh_event event) {
> > +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> > +
> > +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > +	while (r) {
> > +		if (timeout <= 0) {
> > +			pr_err("Doesn't get ack from pf.\n");
> > +			r = -ETIME;
> > +			break;
> > +		}
> > +		msleep(1);
> > +		timeout -= 1;
> > +
> > +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> > +	}
> > +
> > +	return r;
> > +}
> > +
> > +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> > +					enum idh_request request)
> > +{
> > +	int r;
> > +
> > +	xgpu_vi_mailbox_trans_msg(adev, request);
> > +
> > +	/* start to poll ack */
> > +	r = xgpu_vi_poll_ack(adev);
> > +	if (r)
> > +		return r;
> > +
> > +	xgpu_vi_mailbox_set_valid(adev, false);
> > +
> > +	/* start to check msg if request is idh_req_gpu_init_access */
> > +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> > +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> > +		if (r)
> > +			return r;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> > +	return xgpu_vi_send_access_requests(adev,
> IDH_REQ_GPU_RESET_ACCESS);
> > +}
> > +
> > +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> > +					   bool init)
> > +{
> > +	enum idh_event event;
> > +
> > +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> IDH_REQ_GPU_FINI_ACCESS;
> > +	return xgpu_vi_send_access_requests(adev, event); }
> > +
> > +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> > +					   bool init)
> > +{
> > +	enum idh_event event;
> > +	int r = 0;
> > +
> > +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> IDH_REL_GPU_FINI_ACCESS;
> > +	r = xgpu_vi_send_access_requests(adev, event);
> > +
> > +	return r;
> > +}
> > +
> > +/* add support mailbox interrupts */
> > +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> > +				   struct amdgpu_irq_src *source,
> > +				   struct amdgpu_iv_entry *entry) {
> > +	DRM_DEBUG("get ack intr and do nothing.\n");
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> > +				       struct amdgpu_irq_src *src,
> > +				       unsigned type,
> > +				       enum amdgpu_interrupt_state state) {
> > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > +
> > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> > +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> 0);
> > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > +
> > +	return 0;
> > +}
> > +
> > +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> > +	struct amdgpu_virt *virt = container_of(work,
> > +					struct amdgpu_virt, flr_work.work);
> > +	struct amdgpu_device *adev = container_of(virt,
> > +					struct amdgpu_device, virt);
> > +	int r = 0;
> > +
> > +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> > +	if (r)
> > +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> > +
> > +	/* TODO: need to restore gfx states */ }
> > +
> > +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> > +				       struct amdgpu_irq_src *src,
> > +				       unsigned type,
> > +				       enum amdgpu_interrupt_state state) {
> > +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> > +
> > +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> > +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> 0);
> > +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> > +
> > +	return 0;
> > +}
> > +
> > +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> > +				   struct amdgpu_irq_src *source,
> > +				   struct amdgpu_iv_entry *entry) {
> > +	int r;
> > +
> > +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> > +	/* do nothing for other msg */
> > +	if (r)
> > +		return 0;
> > +
> > +	/* TODO: need to save gfx states */
> > +	schedule_delayed_work(&adev->virt.flr_work,
> > +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> > +
> > +	return 0;
> > +}
> > +
> > +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs
> = {
> > +	.set = xgpu_vi_set_mailbox_ack_irq,
> > +	.process = xgpu_vi_mailbox_ack_irq,
> > +};
> > +
> > +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs
> = {
> > +	.set = xgpu_vi_set_mailbox_rcv_irq,
> > +	.process = xgpu_vi_mailbox_rcv_irq,
> > +};
> > +
> > +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> > +	adev->virt.ack_irq.num_types = 1;
> > +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> > +	adev->virt.rcv_irq.num_types = 1;
> > +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> > +
> > +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> > +	int r;
> > +
> > +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> > +	if (r)
> > +		return r;
> > +
> > +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> > +	if (r) {
> > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > +		return r;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> > +	int r;
> > +
> > +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> > +	if (r)
> > +		return r;
> > +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> > +	if (r) {
> > +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> > +		return r;
> > +	}
> > +
> > +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> xgpu_vi_mailbox_flr_work);
> > +
> > +	return 0;
> > +}
> > +
> > +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> > +	cancel_delayed_work_sync(&adev->virt.flr_work);
> > +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> > +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> > +
> > +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> > +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> > +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> > +	.reset_gpu		= xgpu_vi_request_reset,
> > +};
> > diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > new file mode 100644
> > index 0000000..fd6216e
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> > @@ -0,0 +1,55 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + */
> > +
> > +#ifndef __MXGPU_VI_H__
> > +#define __MXGPU_VI_H__
> > +
> > +#define VI_MAILBOX_TIMEDOUT	150
> > +#define VI_MAILBOX_RESET_TIME	12
> > +
> > +/* VI mailbox messages request */
> > +enum idh_request {
> > +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> > +	IDH_REL_GPU_INIT_ACCESS,
> > +	IDH_REQ_GPU_FINI_ACCESS,
> > +	IDH_REL_GPU_FINI_ACCESS,
> > +	IDH_REQ_GPU_RESET_ACCESS
> > +};
> > +
> > +/* VI mailbox messages data */
> > +enum idh_event {
> > +	IDH_CLR_MSG_BUF = 0,
> > +	IDH_READY_TO_ACCESS_GPU,
> > +	IDH_FLR_NOTIFICATION,
> > +	IDH_FLR_NOTIFICATION_CMPL,
> > +	IDH_EVENT_MAX
> > +};
> > +
> > +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> > +
> > +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
> > +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
> > +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> > +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> > +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> > +
> > +#endif
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Christian König]

Am 10.01.2017 um 14:20 schrieb Yu, Xiangliang:
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Tuesday, January 10, 2017 9:16 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
>> gfx@lists.freedesktop.org
>> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
>> interfaces
>>
>> Unrelated to the patch, but what editor and settings do you use?
> Vim

dito.

>
>> See this code for example
>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
>>> +				 TRN_MSG_VALID, val ? 1 : 0);
>> The indentation seems to be incorrect.
> In my platform, there is no problem.

Are you using linuxsty.vim or something else?

If I'm not completely mistaken "TRN_MSG_VALID" should be on the same 
indentation as "reg" and not as "MAILBOX_CONTROL".

Regards,
Christian.

>
>> Not much of a problem, but if we add this like it is anybody else editing the
>> file could mess up the indentation when the editor does that differently.
>>
>> Regards,
>> Christian.
>>
>> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
>>> VI has asic specific virt support, which including mailbox and golden
>>> registers init.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
>>>    drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
>> +++++++++++++++++++++++++++++++
>>>    drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>>>    4 files changed, 651 insertions(+), 1 deletion(-)
>>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 0b8e470..388e7b0 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
>> cik_ih.o kv_smc.o kv_dpm.o \
>>>    amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o
>> si_ih.o
>>> si_dma.o dce_v6_0.o si_dpm.o si_smc.o
>>>
>>>    amdgpu-y += \
>>> -	vi.o
>>> +	vi.o mxgpu_vi.o
>>>
>>>    # add GMC block
>>>    amdgpu-y += \
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 3f8fc0f..611676b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -45,6 +45,9 @@ struct amdgpu_virt {
>>>    	uint32_t		val_offs;
>>>    	struct mutex		lock;
>>>
>>> +	struct amdgpu_irq_src		ack_irq;
>>> +	struct amdgpu_irq_src		rcv_irq;
>>> +	struct delayed_work		flr_work;
>>>    	const struct amdgpu_virt_ops	*ops;
>>>    };
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>> new file mode 100644
>>> index 0000000..7d44663
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>> @@ -0,0 +1,592 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> +the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>> KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
>> NO EVENT
>>> +SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
>> THE USE
>>> +OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + *
>>> + * Authors: Xiangliang.Yu@amd.com
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "vi.h"
>>> +#include "bif/bif_5_0_d.h"
>>> +#include "bif/bif_5_0_sh_mask.h"
>>> +#include "vid.h"
>>> +#include "gca/gfx_8_0_d.h"
>>> +#include "gca/gfx_8_0_sh_mask.h"
>>> +#include "gmc_v8_0.h"
>>> +#include "gfx_v8_0.h"
>>> +#include "sdma_v3_0.h"
>>> +#include "tonga_ih.h"
>>> +#include "gmc/gmc_8_2_d.h"
>>> +#include "gmc/gmc_8_2_sh_mask.h"
>>> +#include "oss/oss_3_0_d.h"
>>> +#include "oss/oss_3_0_sh_mask.h"
>>> +#include "gca/gfx_8_0_sh_mask.h"
>>> +#include "dce/dce_10_0_d.h"
>>> +#include "dce/dce_10_0_sh_mask.h"
>>> +#include "smu/smu_7_1_3_d.h"
>>> +#include "mxgpu_vi.h"
>>> +
>>> +/* VI golden setting */
>>> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
>>> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
>>> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
>>> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
>>> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
>>> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
>>> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
>>> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
>>> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
>>> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
>>> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
>>> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
>>> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
>>> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
>>> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
>>> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
>>> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
>>> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
>>> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
>>> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
>>> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
>>> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
>>> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
>>> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
>>> +
>>> +static const u32 xgpu_fiji_golden_settings_a10[] = {
>>> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
>>> +
>>> +static const u32 xgpu_fiji_golden_common_all[] = {
>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
>>> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
>>> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
>>> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
>>> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
>>> +
>>> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
>>> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
>>> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
>>> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
>>> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
>>> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
>>> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
>>> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
>>> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
>>> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
>>> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
>>> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
>>> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
>>> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
>>> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
>>> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
>>> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
>>> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
>>> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
>>> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
>>> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
>>> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
>>> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
>>> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
>>> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
>>> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
>>> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
>>> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
>>> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
>>> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
>>> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
>>> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
>>> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
>>> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
>>> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
>>> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
>>> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
>>> +};
>>> +
>>> +static const u32 xgpu_tonga_golden_settings_a11[] = {
>>> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
>>> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
>>> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
>>> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
>>> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
>>> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
>>> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
>>> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
>>> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
>>> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
>>> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
>>> +
>>> +static const u32 xgpu_tonga_golden_common_all[] = {
>>> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
>>> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
>>> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
>>> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
>>> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
>>> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
>>> +
>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
>>> +	switch (adev->asic_type) {
>>> +	case CHIP_FIJI:
>>> +		amdgpu_program_register_sequence(adev,
>>> +						 xgpu_fiji_mgcg_cgcg_init,
>>> +						 (const u32)ARRAY_SIZE(
>>> +						 xgpu_fiji_mgcg_cgcg_init));
>>> +		amdgpu_program_register_sequence(adev,
>>> +
>> xgpu_fiji_golden_settings_a10,
>>> +						 (const u32)ARRAY_SIZE(
>>> +
>> xgpu_fiji_golden_settings_a10));
>>> +		amdgpu_program_register_sequence(adev,
>>> +
>> xgpu_fiji_golden_common_all,
>>> +						 (const u32)ARRAY_SIZE(
>>> +
>> xgpu_fiji_golden_common_all));
>>> +		break;
>>> +	case CHIP_TONGA:
>>> +		amdgpu_program_register_sequence(adev,
>>> +						 xgpu_tonga_mgcg_cgcg_init,
>>> +						 (const u32)ARRAY_SIZE(
>>> +
>> xgpu_tonga_mgcg_cgcg_init));
>>> +		amdgpu_program_register_sequence(adev,
>>> +
>> xgpu_tonga_golden_settings_a11,
>>> +						 (const u32)ARRAY_SIZE(
>>> +
>> xgpu_tonga_golden_settings_a11));
>>> +		amdgpu_program_register_sequence(adev,
>>> +
>> xgpu_tonga_golden_common_all,
>>> +						 (const u32)ARRAY_SIZE(
>>> +
>> xgpu_tonga_golden_common_all));
>>> +		break;
>>> +	default:
>>> +		BUG_ON("Doesn't support chip type.\n");
>>> +		break;
>>> +	}
>>> +}
>>> +
>>> +/*
>>> + * Mailbox communication between GPU hypervisor and VFs  */ static
>>> +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
>>> +	u32 reg;
>>> +
>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
>>> +	WREG32(mmMAILBOX_CONTROL, reg);
>>> +}
>>> +
>>> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
>>> +bool val) {
>>> +	u32 reg;
>>> +
>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
>>> +				 TRN_MSG_VALID, val ? 1 : 0);
>>> +	WREG32(mmMAILBOX_CONTROL, reg);
>>> +}
>>> +
>>> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
>>> +				      enum idh_event event)
>>> +{
>>> +	u32 reg;
>>> +
>>> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
>>> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
>>> +				 MSGBUF_DATA, event);
>>> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
>>> +
>>> +	xgpu_vi_mailbox_set_valid(adev, true); }
>>> +
>>> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
>>> +				   enum idh_event event)
>>> +{
>>> +	u32 reg;
>>> +
>>> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
>>> +	if (reg != event)
>>> +		return -ENOENT;
>>> +
>>> +	/* send ack to PF */
>>> +	xgpu_vi_mailbox_send_ack(adev);
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
>>> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
>>> +	u32 reg;
>>> +
>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>> +	while (!(reg & mask)) {
>>> +		if (timeout <= 0) {
>>> +			pr_err("Doesn't get ack from pf.\n");
>>> +			r = -ETIME;
>>> +			break;
>>> +		}
>>> +		msleep(1);
>>> +		timeout -= 1;
>>> +
>>> +		reg = RREG32(mmMAILBOX_CONTROL);
>>> +	}
>>> +
>>> +	return r;
>>> +}
>>> +
>>> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
>>> +idh_event event) {
>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
>>> +
>>> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
>>> +	while (r) {
>>> +		if (timeout <= 0) {
>>> +			pr_err("Doesn't get ack from pf.\n");
>>> +			r = -ETIME;
>>> +			break;
>>> +		}
>>> +		msleep(1);
>>> +		timeout -= 1;
>>> +
>>> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
>>> +	}
>>> +
>>> +	return r;
>>> +}
>>> +
>>> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
>>> +					enum idh_request request)
>>> +{
>>> +	int r;
>>> +
>>> +	xgpu_vi_mailbox_trans_msg(adev, request);
>>> +
>>> +	/* start to poll ack */
>>> +	r = xgpu_vi_poll_ack(adev);
>>> +	if (r)
>>> +		return r;
>>> +
>>> +	xgpu_vi_mailbox_set_valid(adev, false);
>>> +
>>> +	/* start to check msg if request is idh_req_gpu_init_access */
>>> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
>>> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
>>> +		if (r)
>>> +			return r;
>>> +	}
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
>>> +	return xgpu_vi_send_access_requests(adev,
>> IDH_REQ_GPU_RESET_ACCESS);
>>> +}
>>> +
>>> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
>>> +					   bool init)
>>> +{
>>> +	enum idh_event event;
>>> +
>>> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
>> IDH_REQ_GPU_FINI_ACCESS;
>>> +	return xgpu_vi_send_access_requests(adev, event); }
>>> +
>>> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
>>> +					   bool init)
>>> +{
>>> +	enum idh_event event;
>>> +	int r = 0;
>>> +
>>> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
>> IDH_REL_GPU_FINI_ACCESS;
>>> +	r = xgpu_vi_send_access_requests(adev, event);
>>> +
>>> +	return r;
>>> +}
>>> +
>>> +/* add support mailbox interrupts */
>>> +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
>>> +				   struct amdgpu_irq_src *source,
>>> +				   struct amdgpu_iv_entry *entry) {
>>> +	DRM_DEBUG("get ack intr and do nothing.\n");
>>> +	return 0;
>>> +}
>>> +
>>> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
>>> +				       struct amdgpu_irq_src *src,
>>> +				       unsigned type,
>>> +				       enum amdgpu_interrupt_state state) {
>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
>>> +
>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
>>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
>> 0);
>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
>>> +	struct amdgpu_virt *virt = container_of(work,
>>> +					struct amdgpu_virt, flr_work.work);
>>> +	struct amdgpu_device *adev = container_of(virt,
>>> +					struct amdgpu_device, virt);
>>> +	int r = 0;
>>> +
>>> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
>>> +	if (r)
>>> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
>>> +
>>> +	/* TODO: need to restore gfx states */ }
>>> +
>>> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
>>> +				       struct amdgpu_irq_src *src,
>>> +				       unsigned type,
>>> +				       enum amdgpu_interrupt_state state) {
>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
>>> +
>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
>>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
>> 0);
>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
>>> +				   struct amdgpu_irq_src *source,
>>> +				   struct amdgpu_iv_entry *entry) {
>>> +	int r;
>>> +
>>> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
>>> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
>>> +	/* do nothing for other msg */
>>> +	if (r)
>>> +		return 0;
>>> +
>>> +	/* TODO: need to save gfx states */
>>> +	schedule_delayed_work(&adev->virt.flr_work,
>>> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs
>> = {
>>> +	.set = xgpu_vi_set_mailbox_ack_irq,
>>> +	.process = xgpu_vi_mailbox_ack_irq,
>>> +};
>>> +
>>> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs
>> = {
>>> +	.set = xgpu_vi_set_mailbox_rcv_irq,
>>> +	.process = xgpu_vi_mailbox_rcv_irq,
>>> +};
>>> +
>>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
>>> +	adev->virt.ack_irq.num_types = 1;
>>> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
>>> +	adev->virt.rcv_irq.num_types = 1;
>>> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
>>> +
>>> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
>>> +	int r;
>>> +
>>> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
>>> +	if (r)
>>> +		return r;
>>> +
>>> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
>>> +	if (r) {
>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
>>> +		return r;
>>> +	}
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
>>> +	int r;
>>> +
>>> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
>>> +	if (r)
>>> +		return r;
>>> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
>>> +	if (r) {
>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
>>> +		return r;
>>> +	}
>>> +
>>> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
>> xgpu_vi_mailbox_flr_work);
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
>>> +	cancel_delayed_work_sync(&adev->virt.flr_work);
>>> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
>>> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
>>> +
>>> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
>>> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
>>> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
>>> +	.reset_gpu		= xgpu_vi_request_reset,
>>> +};
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>> new file mode 100644
>>> index 0000000..fd6216e
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>> @@ -0,0 +1,55 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> +the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>> KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
>> NO EVENT
>>> +SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
>> THE USE
>>> +OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#ifndef __MXGPU_VI_H__
>>> +#define __MXGPU_VI_H__
>>> +
>>> +#define VI_MAILBOX_TIMEDOUT	150
>>> +#define VI_MAILBOX_RESET_TIME	12
>>> +
>>> +/* VI mailbox messages request */
>>> +enum idh_request {
>>> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
>>> +	IDH_REL_GPU_INIT_ACCESS,
>>> +	IDH_REQ_GPU_FINI_ACCESS,
>>> +	IDH_REL_GPU_FINI_ACCESS,
>>> +	IDH_REQ_GPU_RESET_ACCESS
>>> +};
>>> +
>>> +/* VI mailbox messages data */
>>> +enum idh_event {
>>> +	IDH_CLR_MSG_BUF = 0,
>>> +	IDH_READY_TO_ACCESS_GPU,
>>> +	IDH_FLR_NOTIFICATION,
>>> +	IDH_FLR_NOTIFICATION_CMPL,
>>> +	IDH_EVENT_MAX
>>> +};
>>> +
>>> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
>>> +
>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
>>> +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
>>> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
>>> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
>>> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
>>> +
>>> +#endif


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:12 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > Add high level interfaces that is not relate to specific asic. So asic
> > files just need to implement the interfaces to support virtualization.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> ++++++++++++++++++++++++++++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
> >   2 files changed, 72 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > index 6520a4e..f32a789 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
> *adev, uint32_t reg, uint32_t v)
> >   		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >   	fence_put(f);
> >   }
> > +
> > +/**
> > + * amdgpu_virt_request_full_gpu() - request full gpu access
> > + * @amdgpu:	amdgpu device.
> > + * @init:	is driver init time.
> > + * When start to init/fini driver, first need to request full gpu access.
> > + * Return: Zero if request success, otherwise will return error.
> > + */
> > +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> > +init) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +
> > +	if (virt->ops && virt->ops->req_full_gpu) {
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->req_full_gpu(adev, init);
> 
> I would be conservative here and request full GPU access first and then clear
> AMDGPU_SRIOV_CAPS_RUNTIME.
> 
> Just in case the function is called concurrently with another thread checking
> the caps.

It can't be used in parallel, the problem you said shouldn't be appear.

> 
> On the other hand req_full_gpu() could need the flag to handle register
> reads/writes correctly, but in this case I would question if we shouldn't add
> special macros for this.

We must clear RUNTIME flag so that can read/write registers with mmio,
Otherwise driver will hang.

> 
> 
> 
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/**
> > + * amdgpu_virt_release_full_gpu() - release full gpu access
> > + * @amdgpu:	amdgpu device.
> > + * @init:	is driver init time.
> > + * When finishing driver init/fini, need to release full gpu access.
> > + * Return: Zero if release success, otherwise will returen error.
> > + */
> > +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool
> > +init) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +	int r;
> > +
> > +	if (virt->ops && virt->ops->rel_full_gpu) {
> > +		r = virt->ops->rel_full_gpu(adev, init);
> > +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return r;
> > +	}
> > +	return 0;
> > +}
> > +
> > +/**
> > + * amdgpu_virt_reset_gpu() - reset gpu
> > + * @amdgpu:	amdgpu device.
> > + * Send reset command to GPU hypervisor to reset GPU that VM is using
> > + * Return: Zero if reset success, otherwise will return error.
> > + */
> > +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> > +	struct amdgpu_virt *virt = &adev->virt;
> > +
> > +	if (virt->ops && virt->ops->reset_gpu) {
> > +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> > +		return virt->ops->reset_gpu(adev);
> > +	}
> > +
> > +	return 0;
> > +}
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 24f0590..3f8fc0f 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -29,11 +29,23 @@
> >   #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
> >   #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU
> is pass through for VM */
> >   #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full
> access mode */
> > +
> > +/**
> > + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
> > +amdgpu_virt_ops {
> > +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> > +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> > +	int (*reset_gpu)(struct amdgpu_device *adev); };
> > +
> >   /* GPU virtualization */
> >   struct amdgpu_virt {
> >   	uint32_t		caps;
> >   	uint32_t		val_offs;
> >   	struct mutex		lock;
> > +
> > +	const struct amdgpu_virt_ops	*ops;
> >   };
> >
> >   #define amdgpu_sriov_enabled(adev) \ @@ -63,5 +75,8 @@ static inline
> > bool is_virtual_machine(void)
> >   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
> >   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> reg);
> >   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > uint32_t v);
> > +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> > +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
> > +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> >
> >   #endif
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:19 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Liu, Monk <Monk.Liu@amd.com>
> Subject: Re: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization
> 
> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > Current job timeout setting is not fit for virtualization, so it will
> > cause job timeout sometimes. Add workaround for this when timeout
> > happen.
> >
> > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
> >   1 file changed, 3 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > index a6d828c..5705fbd 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > @@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device
> *adev)
> >   	int resched;
> >   	bool need_full_reset;
> >
> > +	if (amdgpu_sriov_vf(adev))
> 
> At bare minimum we should print a warning here and increase or completely
> disable the job timeout when virtualization is enabled.

The timeout function has the warning message, I think don't need more.
The patch is just a workaround, later need to enhance the whole gpu reset process.
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:30 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
> interfaces
> 
> Am 10.01.2017 um 14:20 schrieb Yu, Xiangliang:
> >> -----Original Message-----
> >> From: Christian König [mailto:deathsimple@vodafone.de]
> >> Sent: Tuesday, January 10, 2017 9:16 PM
> >> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> >> gfx@lists.freedesktop.org
> >> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
> >> interfaces
> >>
> >> Unrelated to the patch, but what editor and settings do you use?
> > Vim
> 
> dito.
> 
> >
> >> See this code for example
> >>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> >>> +				 TRN_MSG_VALID, val ? 1 : 0);
> >> The indentation seems to be incorrect.
> > In my platform, there is no problem.
> 
> Are you using linuxsty.vim or something else?

No, I configure it by myself. 

> 
> If I'm not completely mistaken "TRN_MSG_VALID" should be on the same
> indentation as "reg" and not as "MAILBOX_CONTROL".

I think it should be on same as "MAILBOX_CONTROL" as it is also macro and capital.

> >
> >> Not much of a problem, but if we add this like it is anybody else
> >> editing the file could mess up the indentation when the editor does that
> differently.
> >>
> >> Regards,
> >> Christian.
> >>
> >> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> >>> VI has asic specific virt support, which including mailbox and
> >>> golden registers init.
> >>>
> >>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
> >>>    drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> >> +++++++++++++++++++++++++++++++
> >>>    drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> >>>    4 files changed, 651 insertions(+), 1 deletion(-)
> >>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> index 0b8e470..388e7b0 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> >> cik_ih.o kv_smc.o kv_dpm.o \
> >>>    amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o
> >> si_ih.o
> >>> si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> >>>
> >>>    amdgpu-y += \
> >>> -	vi.o
> >>> +	vi.o mxgpu_vi.o
> >>>
> >>>    # add GMC block
> >>>    amdgpu-y += \
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> index 3f8fc0f..611676b 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> @@ -45,6 +45,9 @@ struct amdgpu_virt {
> >>>    	uint32_t		val_offs;
> >>>    	struct mutex		lock;
> >>>
> >>> +	struct amdgpu_irq_src		ack_irq;
> >>> +	struct amdgpu_irq_src		rcv_irq;
> >>> +	struct delayed_work		flr_work;
> >>>    	const struct amdgpu_virt_ops	*ops;
> >>>    };
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>> new file mode 100644
> >>> index 0000000..7d44663
> >>> --- /dev/null
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>> @@ -0,0 +1,592 @@
> >>> +/*
> >>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>> + *
> >>> + * Permission is hereby granted, free of charge, to any person
> >>> +obtaining a
> >>> + * copy of this software and associated documentation files (the
> >>> +"Software"),
> >>> + * to deal in the Software without restriction, including without
> >>> +limitation
> >>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>> +sublicense,
> >>> + * and/or sell copies of the Software, and to permit persons to
> >>> +whom the
> >>> + * Software is furnished to do so, subject to the following conditions:
> >>> + *
> >>> + * The above copyright notice and this permission notice shall be
> >>> +included in
> >>> + * all copies or substantial portions of the Software.
> >>> + *
> >>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> >> KIND,
> >>> +EXPRESS OR
> >>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>> +MERCHANTABILITY,
> >>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> >> NO EVENT
> >>> +SHALL
> >>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> >>> +DAMAGES OR
> >>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> >>> +OTHERWISE,
> >>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> OR
> >> THE USE
> >>> +OR
> >>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>> + *
> >>> + * Authors: Xiangliang.Yu@amd.com
> >>> + */
> >>> +
> >>> +#include "amdgpu.h"
> >>> +#include "vi.h"
> >>> +#include "bif/bif_5_0_d.h"
> >>> +#include "bif/bif_5_0_sh_mask.h"
> >>> +#include "vid.h"
> >>> +#include "gca/gfx_8_0_d.h"
> >>> +#include "gca/gfx_8_0_sh_mask.h"
> >>> +#include "gmc_v8_0.h"
> >>> +#include "gfx_v8_0.h"
> >>> +#include "sdma_v3_0.h"
> >>> +#include "tonga_ih.h"
> >>> +#include "gmc/gmc_8_2_d.h"
> >>> +#include "gmc/gmc_8_2_sh_mask.h"
> >>> +#include "oss/oss_3_0_d.h"
> >>> +#include "oss/oss_3_0_sh_mask.h"
> >>> +#include "gca/gfx_8_0_sh_mask.h"
> >>> +#include "dce/dce_10_0_d.h"
> >>> +#include "dce/dce_10_0_sh_mask.h"
> >>> +#include "smu/smu_7_1_3_d.h"
> >>> +#include "mxgpu_vi.h"
> >>> +
> >>> +/* VI golden setting */
> >>> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> >>> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> >>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> >>> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> >>> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> >>> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> >>> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> >>> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> >>> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> >>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> >>> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> >>> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> >>> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> >>> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> >>> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> >>> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> >>> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> >>> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> >>> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> >>> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> >>> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> >>> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> >>> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> >>> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> >>> +
> >>> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> >>> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> >>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> >>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> >>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> >>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> >>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> >>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> >>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> >>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> >>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> >>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> >>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> >>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> >>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> >>> +
> >>> +static const u32 xgpu_fiji_golden_common_all[] = {
> >>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> >>> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> >>> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> >>> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> >>> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> >>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> >>> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> >>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> >>> +
> >>> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> >>> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> >>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> >>> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> >>> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> >>> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> >>> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> >>> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> >>> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> >>> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> >>> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> >>> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> >>> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> >>> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> >>> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> >>> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> >>> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> >>> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> >>> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> >>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> >>> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> >>> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> >>> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> >>> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> >>> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> >>> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> >>> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> >>> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> >>> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> >>> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> >>> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> >>> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> >>> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> >>> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> >>> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> >>> +};
> >>> +
> >>> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> >>> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> >>> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> >>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> >>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> >>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> >>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> >>> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> >>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> >>> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> >>> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> >>> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> >>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> >>> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> >>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> >>> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> >>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> >>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> >>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> >>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> >>> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> >>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> >>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> >>> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> >>> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> >>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> >>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> >>> +
> >>> +static const u32 xgpu_tonga_golden_common_all[] = {
> >>> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> >>> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> >>> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> >>> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> >>> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> >>> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> >>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
> >>> +
> >>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> >>> +	switch (adev->asic_type) {
> >>> +	case CHIP_FIJI:
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +						 xgpu_fiji_mgcg_cgcg_init,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +						 xgpu_fiji_mgcg_cgcg_init));
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +
> >> xgpu_fiji_golden_settings_a10,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +
> >> xgpu_fiji_golden_settings_a10));
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +
> >> xgpu_fiji_golden_common_all,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +
> >> xgpu_fiji_golden_common_all));
> >>> +		break;
> >>> +	case CHIP_TONGA:
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +						 xgpu_tonga_mgcg_cgcg_init,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +
> >> xgpu_tonga_mgcg_cgcg_init));
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +
> >> xgpu_tonga_golden_settings_a11,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +
> >> xgpu_tonga_golden_settings_a11));
> >>> +		amdgpu_program_register_sequence(adev,
> >>> +
> >> xgpu_tonga_golden_common_all,
> >>> +						 (const u32)ARRAY_SIZE(
> >>> +
> >> xgpu_tonga_golden_common_all));
> >>> +		break;
> >>> +	default:
> >>> +		BUG_ON("Doesn't support chip type.\n");
> >>> +		break;
> >>> +	}
> >>> +}
> >>> +
> >>> +/*
> >>> + * Mailbox communication between GPU hypervisor and VFs  */ static
> >>> +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> >>> +	u32 reg;
> >>> +
> >>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> >>> +	WREG32(mmMAILBOX_CONTROL, reg);
> >>> +}
> >>> +
> >>> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
> >>> +bool val) {
> >>> +	u32 reg;
> >>> +
> >>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> >>> +				 TRN_MSG_VALID, val ? 1 : 0);
> >>> +	WREG32(mmMAILBOX_CONTROL, reg);
> >>> +}
> >>> +
> >>> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> >>> +				      enum idh_event event)
> >>> +{
> >>> +	u32 reg;
> >>> +
> >>> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> >>> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> >>> +				 MSGBUF_DATA, event);
> >>> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> >>> +
> >>> +	xgpu_vi_mailbox_set_valid(adev, true); }
> >>> +
> >>> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> >>> +				   enum idh_event event)
> >>> +{
> >>> +	u32 reg;
> >>> +
> >>> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> >>> +	if (reg != event)
> >>> +		return -ENOENT;
> >>> +
> >>> +	/* send ack to PF */
> >>> +	xgpu_vi_mailbox_send_ack(adev);
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> >>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> >>> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
> >>> +	u32 reg;
> >>> +
> >>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>> +	while (!(reg & mask)) {
> >>> +		if (timeout <= 0) {
> >>> +			pr_err("Doesn't get ack from pf.\n");
> >>> +			r = -ETIME;
> >>> +			break;
> >>> +		}
> >>> +		msleep(1);
> >>> +		timeout -= 1;
> >>> +
> >>> +		reg = RREG32(mmMAILBOX_CONTROL);
> >>> +	}
> >>> +
> >>> +	return r;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
> >>> +idh_event event) {
> >>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> >>> +
> >>> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> >>> +	while (r) {
> >>> +		if (timeout <= 0) {
> >>> +			pr_err("Doesn't get ack from pf.\n");
> >>> +			r = -ETIME;
> >>> +			break;
> >>> +		}
> >>> +		msleep(1);
> >>> +		timeout -= 1;
> >>> +
> >>> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> >>> +	}
> >>> +
> >>> +	return r;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> >>> +					enum idh_request request)
> >>> +{
> >>> +	int r;
> >>> +
> >>> +	xgpu_vi_mailbox_trans_msg(adev, request);
> >>> +
> >>> +	/* start to poll ack */
> >>> +	r = xgpu_vi_poll_ack(adev);
> >>> +	if (r)
> >>> +		return r;
> >>> +
> >>> +	xgpu_vi_mailbox_set_valid(adev, false);
> >>> +
> >>> +	/* start to check msg if request is idh_req_gpu_init_access */
> >>> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> >>> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> >>> +		if (r)
> >>> +			return r;
> >>> +	}
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> >>> +	return xgpu_vi_send_access_requests(adev,
> >> IDH_REQ_GPU_RESET_ACCESS);
> >>> +}
> >>> +
> >>> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device
> *adev,
> >>> +					   bool init)
> >>> +{
> >>> +	enum idh_event event;
> >>> +
> >>> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> >> IDH_REQ_GPU_FINI_ACCESS;
> >>> +	return xgpu_vi_send_access_requests(adev, event); }
> >>> +
> >>> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device
> *adev,
> >>> +					   bool init)
> >>> +{
> >>> +	enum idh_event event;
> >>> +	int r = 0;
> >>> +
> >>> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> >> IDH_REL_GPU_FINI_ACCESS;
> >>> +	r = xgpu_vi_send_access_requests(adev, event);
> >>> +
> >>> +	return r;
> >>> +}
> >>> +
> >>> +/* add support mailbox interrupts */ static int
> >>> +xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> >>> +				   struct amdgpu_irq_src *source,
> >>> +				   struct amdgpu_iv_entry *entry) {
> >>> +	DRM_DEBUG("get ack intr and do nothing.\n");
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> >>> +				       struct amdgpu_irq_src *src,
> >>> +				       unsigned type,
> >>> +				       enum amdgpu_interrupt_state state) {
> >>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> >>> +
> >>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> >>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> >> 0);
> >>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> >>> +	struct amdgpu_virt *virt = container_of(work,
> >>> +					struct amdgpu_virt, flr_work.work);
> >>> +	struct amdgpu_device *adev = container_of(virt,
> >>> +					struct amdgpu_device, virt);
> >>> +	int r = 0;
> >>> +
> >>> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> >>> +	if (r)
> >>> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> >>> +
> >>> +	/* TODO: need to restore gfx states */ }
> >>> +
> >>> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> >>> +				       struct amdgpu_irq_src *src,
> >>> +				       unsigned type,
> >>> +				       enum amdgpu_interrupt_state state) {
> >>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> >>> +
> >>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> >>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> >> 0);
> >>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> >>> +				   struct amdgpu_irq_src *source,
> >>> +				   struct amdgpu_iv_entry *entry) {
> >>> +	int r;
> >>> +
> >>> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> >>> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> >>> +	/* do nothing for other msg */
> >>> +	if (r)
> >>> +		return 0;
> >>> +
> >>> +	/* TODO: need to save gfx states */
> >>> +	schedule_delayed_work(&adev->virt.flr_work,
> >>> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +static const struct amdgpu_irq_src_funcs
> >>> +xgpu_vi_mailbox_ack_irq_funcs
> >> = {
> >>> +	.set = xgpu_vi_set_mailbox_ack_irq,
> >>> +	.process = xgpu_vi_mailbox_ack_irq, };
> >>> +
> >>> +static const struct amdgpu_irq_src_funcs
> >>> +xgpu_vi_mailbox_rcv_irq_funcs
> >> = {
> >>> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> >>> +	.process = xgpu_vi_mailbox_rcv_irq, };
> >>> +
> >>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
> >>> +	adev->virt.ack_irq.num_types = 1;
> >>> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> >>> +	adev->virt.rcv_irq.num_types = 1;
> >>> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> >>> +
> >>> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> >>> +	int r;
> >>> +
> >>> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> >>> +	if (r)
> >>> +		return r;
> >>> +
> >>> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> >>> +	if (r) {
> >>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> >>> +		return r;
> >>> +	}
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> >>> +	int r;
> >>> +
> >>> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> >>> +	if (r)
> >>> +		return r;
> >>> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> >>> +	if (r) {
> >>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> >>> +		return r;
> >>> +	}
> >>> +
> >>> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> >> xgpu_vi_mailbox_flr_work);
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> >>> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> >>> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> >>> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> >>> +
> >>> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> >>> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> >>> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> >>> +	.reset_gpu		= xgpu_vi_request_reset,
> >>> +};
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>> new file mode 100644
> >>> index 0000000..fd6216e
> >>> --- /dev/null
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>> @@ -0,0 +1,55 @@
> >>> +/*
> >>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>> + *
> >>> + * Permission is hereby granted, free of charge, to any person
> >>> +obtaining a
> >>> + * copy of this software and associated documentation files (the
> >>> +"Software"),
> >>> + * to deal in the Software without restriction, including without
> >>> +limitation
> >>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>> +sublicense,
> >>> + * and/or sell copies of the Software, and to permit persons to
> >>> +whom the
> >>> + * Software is furnished to do so, subject to the following conditions:
> >>> + *
> >>> + * The above copyright notice and this permission notice shall be
> >>> +included in
> >>> + * all copies or substantial portions of the Software.
> >>> + *
> >>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> >> KIND,
> >>> +EXPRESS OR
> >>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>> +MERCHANTABILITY,
> >>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> >> NO EVENT
> >>> +SHALL
> >>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> >>> +DAMAGES OR
> >>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> >>> +OTHERWISE,
> >>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> OR
> >> THE USE
> >>> +OR
> >>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>> + */
> >>> +
> >>> +#ifndef __MXGPU_VI_H__
> >>> +#define __MXGPU_VI_H__
> >>> +
> >>> +#define VI_MAILBOX_TIMEDOUT	150
> >>> +#define VI_MAILBOX_RESET_TIME	12
> >>> +
> >>> +/* VI mailbox messages request */
> >>> +enum idh_request {
> >>> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> >>> +	IDH_REL_GPU_INIT_ACCESS,
> >>> +	IDH_REQ_GPU_FINI_ACCESS,
> >>> +	IDH_REL_GPU_FINI_ACCESS,
> >>> +	IDH_REQ_GPU_RESET_ACCESS
> >>> +};
> >>> +
> >>> +/* VI mailbox messages data */
> >>> +enum idh_event {
> >>> +	IDH_CLR_MSG_BUF = 0,
> >>> +	IDH_READY_TO_ACCESS_GPU,
> >>> +	IDH_FLR_NOTIFICATION,
> >>> +	IDH_FLR_NOTIFICATION_CMPL,
> >>> +	IDH_EVENT_MAX
> >>> +};
> >>> +
> >>> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> >>> +
> >>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
> >>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
> >>> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> >>> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> >>> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> >>> +
> >>> +#endif
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Christian König]

Am 10.01.2017 um 14:33 schrieb Yu, Xiangliang:
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Tuesday, January 10, 2017 9:12 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
>> gfx@lists.freedesktop.org
>> Subject: Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt
>>
>> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
>>> Add high level interfaces that is not relate to specific asic. So asic
>>> files just need to implement the interfaces to support virtualization.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
>> ++++++++++++++++++++++++++++++++
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
>>>    2 files changed, 72 insertions(+)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> index 6520a4e..f32a789 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
>> *adev, uint32_t reg, uint32_t v)
>>>    		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>>    	fence_put(f);
>>>    }
>>> +
>>> +/**
>>> + * amdgpu_virt_request_full_gpu() - request full gpu access
>>> + * @amdgpu:	amdgpu device.
>>> + * @init:	is driver init time.
>>> + * When start to init/fini driver, first need to request full gpu access.
>>> + * Return: Zero if request success, otherwise will return error.
>>> + */
>>> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
>>> +init) {
>>> +	struct amdgpu_virt *virt = &adev->virt;
>>> +
>>> +	if (virt->ops && virt->ops->req_full_gpu) {
>>> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
>>> +		return virt->ops->req_full_gpu(adev, init);
>> I would be conservative here and request full GPU access first and then clear
>> AMDGPU_SRIOV_CAPS_RUNTIME.
>>
>> Just in case the function is called concurrently with another thread checking
>> the caps.
> It can't be used in parallel, the problem you said shouldn't be appear.
>
>> On the other hand req_full_gpu() could need the flag to handle register
>> reads/writes correctly, but in this case I would question if we shouldn't add
>> special macros for this.
> We must clear RUNTIME flag so that can read/write registers with mmio,
> Otherwise driver will hang.

Yeah, that's what I expected. Would be nice if we could better split 
that, e.g. have explicit macros for direct register write.

But that's really only nice to have, patch is ok as it is as far as I 
can see.

Christian.

>
>>
>>
>>> +	}
>>> +
>>> +	return 0;
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_virt_release_full_gpu() - release full gpu access
>>> + * @amdgpu:	amdgpu device.
>>> + * @init:	is driver init time.
>>> + * When finishing driver init/fini, need to release full gpu access.
>>> + * Return: Zero if release success, otherwise will returen error.
>>> + */
>>> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool
>>> +init) {
>>> +	struct amdgpu_virt *virt = &adev->virt;
>>> +	int r;
>>> +
>>> +	if (virt->ops && virt->ops->rel_full_gpu) {
>>> +		r = virt->ops->rel_full_gpu(adev, init);
>>> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
>>> +		return r;
>>> +	}
>>> +	return 0;
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_virt_reset_gpu() - reset gpu
>>> + * @amdgpu:	amdgpu device.
>>> + * Send reset command to GPU hypervisor to reset GPU that VM is using
>>> + * Return: Zero if reset success, otherwise will return error.
>>> + */
>>> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
>>> +	struct amdgpu_virt *virt = &adev->virt;
>>> +
>>> +	if (virt->ops && virt->ops->reset_gpu) {
>>> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
>>> +		return virt->ops->reset_gpu(adev);
>>> +	}
>>> +
>>> +	return 0;
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 24f0590..3f8fc0f 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -29,11 +29,23 @@
>>>    #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
>> function */
>>>    #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU
>> is pass through for VM */
>>>    #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full
>> access mode */
>>> +
>>> +/**
>>> + * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
>>> +amdgpu_virt_ops {
>>> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
>>> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
>>> +	int (*reset_gpu)(struct amdgpu_device *adev); };
>>> +
>>>    /* GPU virtualization */
>>>    struct amdgpu_virt {
>>>    	uint32_t		caps;
>>>    	uint32_t		val_offs;
>>>    	struct mutex		lock;
>>> +
>>> +	const struct amdgpu_virt_ops	*ops;
>>>    };
>>>
>>>    #define amdgpu_sriov_enabled(adev) \ @@ -63,5 +75,8 @@ static inline
>>> bool is_virtual_machine(void)
>>>    void amdgpu_virt_init_setting(struct amdgpu_device *adev);
>>>    uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>> reg);
>>>    void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> uint32_t v);
>>> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
>>> +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
>>> +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
>>>
>>>    #endif


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:09 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
> 
> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > For virtualization, it is must for driver to use KIQ to access
> > registers when it is out of GPU full access mode.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86
> ++++++++++++++++++++++++++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
> >   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
> >   5 files changed, 101 insertions(+), 1 deletion(-)
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > index 4185b03..0b8e470 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
> >   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
> >   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
> >   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> > -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> > +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
> >
> >   # add asic specific block
> >   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
> kv_dpm.o \
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > index f82919d..9a2fd3e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device
> *adev, uint32_t reg,
> >   {
> >   	uint32_t ret;
> >
> > +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> > +		return amdgpu_virt_kiq_rreg(adev, reg);
> > +
> >   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
> >   	else {
> > @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device
> *adev, uint32_t reg, uint32_t v,
> >   {
> >   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
> >
> > +	if (amdgpu_sriov_runtime(adev))
> > +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> > +
> >   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
> >   	else {
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > new file mode 100644
> > index 0000000..6520a4e
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -0,0 +1,86 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + */
> > +
> > +#include "amdgpu.h"
> > +#include "amdgpu_virt.h"
> > +
> > +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> > +	mutex_init(&adev->virt.lock);
> > +}
> > +
> > +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> > +reg) {
> > +	signed long r;
> > +	uint32_t val;
> > +	struct fence *f;
> > +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > +	struct amdgpu_ring *ring = &kiq->ring;
> > +
> > +	BUG_ON(!ring->funcs->emit_rreg);
> > +
> > +	mutex_lock(&adev->virt.lock);
> > +	amdgpu_ring_alloc(ring, 32);
> > +	amdgpu_ring_emit_hdp_flush(ring);
> > +	amdgpu_ring_emit_rreg(ring, reg);
> > +	amdgpu_ring_emit_hdp_invalidate(ring);
> > +	amdgpu_fence_emit(ring, &f);
> > +	fence_get(f);
> > +	amdgpu_ring_commit(ring);
> > +	mutex_unlock(&adev->virt.lock);
> > +
> > +	r = fence_wait(f, false);
> > +	fence_put(f);
> 
> Why do you grab and release an extra fence reference here?

KIQ ring has no scheduler, I think we need to take care fence.

> 
> > +	if (r)
> > +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> > +	fence_put(f);
> > +
> > +	val = adev->wb.wb[adev->virt.val_offs];
> > +
> > +	return val;
> > +}
> > +
> > +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > +uint32_t v) {
> > +	signed long r;
> > +	struct fence *f;
> > +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > +	struct amdgpu_ring *ring = &kiq->ring;
> > +
> > +	BUG_ON(!ring->funcs->emit_wreg);
> > +
> > +	mutex_lock(&adev->virt.lock);
> > +	amdgpu_ring_alloc(ring, 32);
> > +	amdgpu_ring_emit_hdp_flush(ring);
> > +	amdgpu_ring_emit_wreg(ring, reg, v);
> > +	amdgpu_ring_emit_hdp_invalidate(ring);
> > +	amdgpu_fence_emit(ring, &f);
> > +	fence_get(f);
> > +	amdgpu_ring_commit(ring);
> > +	mutex_unlock(&adev->virt.lock);
> > +
> > +	r = fence_wait(f, false);
> > +	fence_put(f);
> > +	if (r)
> > +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> > +	fence_put(f);
> > +}
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 79619b6..24f0590 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -33,6 +33,7 @@
> >   struct amdgpu_virt {
> >   	uint32_t		caps;
> >   	uint32_t		val_offs;
> > +	struct mutex		lock;
> >   };
> >
> >   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline
> > bool is_virtual_machine(void)
> >   #endif
> >   }
> >
> > +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
> > +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
> > +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > +uint32_t v);
> > +
> >   #endif
> > diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> > b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> > @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
> >   		(amdgpu_ip_block_mask & (1 <<
> AMD_IP_BLOCK_TYPE_SMC)))
> >   		smc_enabled = true;
> >
> > +	if (amdgpu_sriov_vf(adev))
> > +		amdgpu_virt_init_setting(adev);
> > +
> >   	adev->rev_id = vi_get_rev_id(adev);
> >   	adev->external_rev_id = 0xFF;
> >   	switch (adev->asic_type) {
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Christian König]

Am 10.01.2017 um 14:43 schrieb Yu, Xiangliang:
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Tuesday, January 10, 2017 9:30 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
>> gfx@lists.freedesktop.org
>> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
>> interfaces
>>
>> Am 10.01.2017 um 14:20 schrieb Yu, Xiangliang:
>>>> -----Original Message-----
>>>> From: Christian König [mailto:deathsimple@vodafone.de]
>>>> Sent: Tuesday, January 10, 2017 9:16 PM
>>>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
>>>> gfx@lists.freedesktop.org
>>>> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
>>>> interfaces
>>>>
>>>> Unrelated to the patch, but what editor and settings do you use?
>>> Vim
>> dito.
>>
>>>> See this code for example
>>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
>>>>> +				 TRN_MSG_VALID, val ? 1 : 0);
>>>> The indentation seems to be incorrect.
>>> In my platform, there is no problem.
>> Are you using linuxsty.vim or something else?
> No, I configure it by myself.

Ah, there it comes from. Please use linuxsty.vim or any other predefined 
set of setting for vim.

Configuring it yourself is usually not a good idea, cause that can 
easily lead to slightly different coding styles.


>
>> If I'm not completely mistaken "TRN_MSG_VALID" should be on the same
>> indentation as "reg" and not as "MAILBOX_CONTROL".
> I think it should be on same as "MAILBOX_CONTROL" as it is also macro and capital.

No, function arguments broken to the next line should start at the same 
place as the "(" of the previous line.

See the other code in amdgpu as well or just use linuxsty.

Regards,
Christian.

>
>>>> Not much of a problem, but if we add this like it is anybody else
>>>> editing the file could mess up the indentation when the editor does that
>> differently.
>>>> Regards,
>>>> Christian.
>>>>
>>>> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
>>>>> VI has asic specific virt support, which including mailbox and
>>>>> golden registers init.
>>>>>
>>>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>>>> ---
>>>>>     drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>>>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
>>>>>     drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
>>>> +++++++++++++++++++++++++++++++
>>>>>     drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>>>>>     4 files changed, 651 insertions(+), 1 deletion(-)
>>>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>>>>
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>>>> index 0b8e470..388e7b0 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>>>> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
>>>> cik_ih.o kv_smc.o kv_dpm.o \
>>>>>     amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o
>>>> si_ih.o
>>>>> si_dma.o dce_v6_0.o si_dpm.o si_smc.o
>>>>>
>>>>>     amdgpu-y += \
>>>>> -	vi.o
>>>>> +	vi.o mxgpu_vi.o
>>>>>
>>>>>     # add GMC block
>>>>>     amdgpu-y += \
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>>>> index 3f8fc0f..611676b 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>>>> @@ -45,6 +45,9 @@ struct amdgpu_virt {
>>>>>     	uint32_t		val_offs;
>>>>>     	struct mutex		lock;
>>>>>
>>>>> +	struct amdgpu_irq_src		ack_irq;
>>>>> +	struct amdgpu_irq_src		rcv_irq;
>>>>> +	struct delayed_work		flr_work;
>>>>>     	const struct amdgpu_virt_ops	*ops;
>>>>>     };
>>>>>
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>>>> new file mode 100644
>>>>> index 0000000..7d44663
>>>>> --- /dev/null
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>>>>> @@ -0,0 +1,592 @@
>>>>> +/*
>>>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>>>> + *
>>>>> + * Permission is hereby granted, free of charge, to any person
>>>>> +obtaining a
>>>>> + * copy of this software and associated documentation files (the
>>>>> +"Software"),
>>>>> + * to deal in the Software without restriction, including without
>>>>> +limitation
>>>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>>>> +sublicense,
>>>>> + * and/or sell copies of the Software, and to permit persons to
>>>>> +whom the
>>>>> + * Software is furnished to do so, subject to the following conditions:
>>>>> + *
>>>>> + * The above copyright notice and this permission notice shall be
>>>>> +included in
>>>>> + * all copies or substantial portions of the Software.
>>>>> + *
>>>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>>>> KIND,
>>>>> +EXPRESS OR
>>>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>>>> +MERCHANTABILITY,
>>>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
>>>> NO EVENT
>>>>> +SHALL
>>>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>>>> +DAMAGES OR
>>>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>>>> +OTHERWISE,
>>>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
>> OR
>>>> THE USE
>>>>> +OR
>>>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>>>> + *
>>>>> + * Authors: Xiangliang.Yu@amd.com
>>>>> + */
>>>>> +
>>>>> +#include "amdgpu.h"
>>>>> +#include "vi.h"
>>>>> +#include "bif/bif_5_0_d.h"
>>>>> +#include "bif/bif_5_0_sh_mask.h"
>>>>> +#include "vid.h"
>>>>> +#include "gca/gfx_8_0_d.h"
>>>>> +#include "gca/gfx_8_0_sh_mask.h"
>>>>> +#include "gmc_v8_0.h"
>>>>> +#include "gfx_v8_0.h"
>>>>> +#include "sdma_v3_0.h"
>>>>> +#include "tonga_ih.h"
>>>>> +#include "gmc/gmc_8_2_d.h"
>>>>> +#include "gmc/gmc_8_2_sh_mask.h"
>>>>> +#include "oss/oss_3_0_d.h"
>>>>> +#include "oss/oss_3_0_sh_mask.h"
>>>>> +#include "gca/gfx_8_0_sh_mask.h"
>>>>> +#include "dce/dce_10_0_d.h"
>>>>> +#include "dce/dce_10_0_sh_mask.h"
>>>>> +#include "smu/smu_7_1_3_d.h"
>>>>> +#include "mxgpu_vi.h"
>>>>> +
>>>>> +/* VI golden setting */
>>>>> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
>>>>> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
>>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>>>> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
>>>>> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
>>>>> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
>>>>> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
>>>>> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
>>>>> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
>>>>> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
>>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>>>> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
>>>>> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
>>>>> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
>>>>> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
>>>>> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
>>>>> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
>>>>> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
>>>>> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
>>>>> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
>>>>> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
>>>>> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
>>>>> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
>>>>> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
>>>>> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
>>>>> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
>>>>> +
>>>>> +static const u32 xgpu_fiji_golden_settings_a10[] = {
>>>>> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
>>>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
>>>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
>>>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
>>>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
>>>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
>>>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
>>>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
>>>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
>>>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
>>>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
>>>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
>>>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
>>>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
>>>>> +
>>>>> +static const u32 xgpu_fiji_golden_common_all[] = {
>>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>>>> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
>>>>> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
>>>>> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
>>>>> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
>>>>> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
>>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
>>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
>>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
>>>>> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
>>>>> +
>>>>> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
>>>>> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
>>>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
>>>>> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
>>>>> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
>>>>> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
>>>>> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
>>>>> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
>>>>> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
>>>>> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
>>>>> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
>>>>> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
>>>>> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
>>>>> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
>>>>> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
>>>>> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
>>>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
>>>>> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
>>>>> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
>>>>> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
>>>>> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
>>>>> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
>>>>> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
>>>>> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
>>>>> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
>>>>> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
>>>>> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
>>>>> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
>>>>> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
>>>>> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
>>>>> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
>>>>> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
>>>>> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
>>>>> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
>>>>> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
>>>>> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
>>>>> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
>>>>> +};
>>>>> +
>>>>> +static const u32 xgpu_tonga_golden_settings_a11[] = {
>>>>> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
>>>>> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
>>>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
>>>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
>>>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
>>>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
>>>>> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
>>>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
>>>>> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
>>>>> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
>>>>> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
>>>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
>>>>> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
>>>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
>>>>> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
>>>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>>>> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
>>>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
>>>>> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
>>>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
>>>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
>>>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
>>>>> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
>>>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
>>>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
>>>>> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
>>>>> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
>>>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
>>>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
>>>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
>>>>> +
>>>>> +static const u32 xgpu_tonga_golden_common_all[] = {
>>>>> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
>>>>> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
>>>>> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
>>>>> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
>>>>> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
>>>>> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
>>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
>>>>> +
>>>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
>>>>> +	switch (adev->asic_type) {
>>>>> +	case CHIP_FIJI:
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +						 xgpu_fiji_mgcg_cgcg_init,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +						 xgpu_fiji_mgcg_cgcg_init));
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +
>>>> xgpu_fiji_golden_settings_a10,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +
>>>> xgpu_fiji_golden_settings_a10));
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +
>>>> xgpu_fiji_golden_common_all,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +
>>>> xgpu_fiji_golden_common_all));
>>>>> +		break;
>>>>> +	case CHIP_TONGA:
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +						 xgpu_tonga_mgcg_cgcg_init,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +
>>>> xgpu_tonga_mgcg_cgcg_init));
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +
>>>> xgpu_tonga_golden_settings_a11,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +
>>>> xgpu_tonga_golden_settings_a11));
>>>>> +		amdgpu_program_register_sequence(adev,
>>>>> +
>>>> xgpu_tonga_golden_common_all,
>>>>> +						 (const u32)ARRAY_SIZE(
>>>>> +
>>>> xgpu_tonga_golden_common_all));
>>>>> +		break;
>>>>> +	default:
>>>>> +		BUG_ON("Doesn't support chip type.\n");
>>>>> +		break;
>>>>> +	}
>>>>> +}
>>>>> +
>>>>> +/*
>>>>> + * Mailbox communication between GPU hypervisor and VFs  */ static
>>>>> +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
>>>>> +	u32 reg;
>>>>> +
>>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
>>>>> +	WREG32(mmMAILBOX_CONTROL, reg);
>>>>> +}
>>>>> +
>>>>> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
>>>>> +bool val) {
>>>>> +	u32 reg;
>>>>> +
>>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
>>>>> +				 TRN_MSG_VALID, val ? 1 : 0);
>>>>> +	WREG32(mmMAILBOX_CONTROL, reg);
>>>>> +}
>>>>> +
>>>>> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
>>>>> +				      enum idh_event event)
>>>>> +{
>>>>> +	u32 reg;
>>>>> +
>>>>> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
>>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
>>>>> +				 MSGBUF_DATA, event);
>>>>> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
>>>>> +
>>>>> +	xgpu_vi_mailbox_set_valid(adev, true); }
>>>>> +
>>>>> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
>>>>> +				   enum idh_event event)
>>>>> +{
>>>>> +	u32 reg;
>>>>> +
>>>>> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
>>>>> +	if (reg != event)
>>>>> +		return -ENOENT;
>>>>> +
>>>>> +	/* send ack to PF */
>>>>> +	xgpu_vi_mailbox_send_ack(adev);
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
>>>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
>>>>> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
>>>>> +	u32 reg;
>>>>> +
>>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
>>>>> +	while (!(reg & mask)) {
>>>>> +		if (timeout <= 0) {
>>>>> +			pr_err("Doesn't get ack from pf.\n");
>>>>> +			r = -ETIME;
>>>>> +			break;
>>>>> +		}
>>>>> +		msleep(1);
>>>>> +		timeout -= 1;
>>>>> +
>>>>> +		reg = RREG32(mmMAILBOX_CONTROL);
>>>>> +	}
>>>>> +
>>>>> +	return r;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
>>>>> +idh_event event) {
>>>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
>>>>> +
>>>>> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
>>>>> +	while (r) {
>>>>> +		if (timeout <= 0) {
>>>>> +			pr_err("Doesn't get ack from pf.\n");
>>>>> +			r = -ETIME;
>>>>> +			break;
>>>>> +		}
>>>>> +		msleep(1);
>>>>> +		timeout -= 1;
>>>>> +
>>>>> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
>>>>> +	}
>>>>> +
>>>>> +	return r;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
>>>>> +					enum idh_request request)
>>>>> +{
>>>>> +	int r;
>>>>> +
>>>>> +	xgpu_vi_mailbox_trans_msg(adev, request);
>>>>> +
>>>>> +	/* start to poll ack */
>>>>> +	r = xgpu_vi_poll_ack(adev);
>>>>> +	if (r)
>>>>> +		return r;
>>>>> +
>>>>> +	xgpu_vi_mailbox_set_valid(adev, false);
>>>>> +
>>>>> +	/* start to check msg if request is idh_req_gpu_init_access */
>>>>> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
>>>>> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
>>>>> +		if (r)
>>>>> +			return r;
>>>>> +	}
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
>>>>> +	return xgpu_vi_send_access_requests(adev,
>>>> IDH_REQ_GPU_RESET_ACCESS);
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device
>> *adev,
>>>>> +					   bool init)
>>>>> +{
>>>>> +	enum idh_event event;
>>>>> +
>>>>> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
>>>> IDH_REQ_GPU_FINI_ACCESS;
>>>>> +	return xgpu_vi_send_access_requests(adev, event); }
>>>>> +
>>>>> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device
>> *adev,
>>>>> +					   bool init)
>>>>> +{
>>>>> +	enum idh_event event;
>>>>> +	int r = 0;
>>>>> +
>>>>> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
>>>> IDH_REL_GPU_FINI_ACCESS;
>>>>> +	r = xgpu_vi_send_access_requests(adev, event);
>>>>> +
>>>>> +	return r;
>>>>> +}
>>>>> +
>>>>> +/* add support mailbox interrupts */ static int
>>>>> +xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
>>>>> +				   struct amdgpu_irq_src *source,
>>>>> +				   struct amdgpu_iv_entry *entry) {
>>>>> +	DRM_DEBUG("get ack intr and do nothing.\n");
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
>>>>> +				       struct amdgpu_irq_src *src,
>>>>> +				       unsigned type,
>>>>> +				       enum amdgpu_interrupt_state state) {
>>>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
>>>>> +
>>>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
>>>>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
>>>> 0);
>>>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
>>>>> +	struct amdgpu_virt *virt = container_of(work,
>>>>> +					struct amdgpu_virt, flr_work.work);
>>>>> +	struct amdgpu_device *adev = container_of(virt,
>>>>> +					struct amdgpu_device, virt);
>>>>> +	int r = 0;
>>>>> +
>>>>> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
>>>>> +	if (r)
>>>>> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
>>>>> +
>>>>> +	/* TODO: need to restore gfx states */ }
>>>>> +
>>>>> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
>>>>> +				       struct amdgpu_irq_src *src,
>>>>> +				       unsigned type,
>>>>> +				       enum amdgpu_interrupt_state state) {
>>>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
>>>>> +
>>>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
>>>>> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 :
>>>> 0);
>>>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
>>>>> +				   struct amdgpu_irq_src *source,
>>>>> +				   struct amdgpu_iv_entry *entry) {
>>>>> +	int r;
>>>>> +
>>>>> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
>>>>> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
>>>>> +	/* do nothing for other msg */
>>>>> +	if (r)
>>>>> +		return 0;
>>>>> +
>>>>> +	/* TODO: need to save gfx states */
>>>>> +	schedule_delayed_work(&adev->virt.flr_work,
>>>>> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +static const struct amdgpu_irq_src_funcs
>>>>> +xgpu_vi_mailbox_ack_irq_funcs
>>>> = {
>>>>> +	.set = xgpu_vi_set_mailbox_ack_irq,
>>>>> +	.process = xgpu_vi_mailbox_ack_irq, };
>>>>> +
>>>>> +static const struct amdgpu_irq_src_funcs
>>>>> +xgpu_vi_mailbox_rcv_irq_funcs
>>>> = {
>>>>> +	.set = xgpu_vi_set_mailbox_rcv_irq,
>>>>> +	.process = xgpu_vi_mailbox_rcv_irq, };
>>>>> +
>>>>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
>>>>> +	adev->virt.ack_irq.num_types = 1;
>>>>> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
>>>>> +	adev->virt.rcv_irq.num_types = 1;
>>>>> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
>>>>> +
>>>>> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
>>>>> +	int r;
>>>>> +
>>>>> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
>>>>> +	if (r)
>>>>> +		return r;
>>>>> +
>>>>> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
>>>>> +	if (r) {
>>>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
>>>>> +		return r;
>>>>> +	}
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
>>>>> +	int r;
>>>>> +
>>>>> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
>>>>> +	if (r)
>>>>> +		return r;
>>>>> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
>>>>> +	if (r) {
>>>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
>>>>> +		return r;
>>>>> +	}
>>>>> +
>>>>> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
>>>> xgpu_vi_mailbox_flr_work);
>>>>> +
>>>>> +	return 0;
>>>>> +}
>>>>> +
>>>>> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
>>>>> +	cancel_delayed_work_sync(&adev->virt.flr_work);
>>>>> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
>>>>> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
>>>>> +
>>>>> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
>>>>> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
>>>>> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
>>>>> +	.reset_gpu		= xgpu_vi_request_reset,
>>>>> +};
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>>>> new file mode 100644
>>>>> index 0000000..fd6216e
>>>>> --- /dev/null
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
>>>>> @@ -0,0 +1,55 @@
>>>>> +/*
>>>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>>>> + *
>>>>> + * Permission is hereby granted, free of charge, to any person
>>>>> +obtaining a
>>>>> + * copy of this software and associated documentation files (the
>>>>> +"Software"),
>>>>> + * to deal in the Software without restriction, including without
>>>>> +limitation
>>>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>>>> +sublicense,
>>>>> + * and/or sell copies of the Software, and to permit persons to
>>>>> +whom the
>>>>> + * Software is furnished to do so, subject to the following conditions:
>>>>> + *
>>>>> + * The above copyright notice and this permission notice shall be
>>>>> +included in
>>>>> + * all copies or substantial portions of the Software.
>>>>> + *
>>>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>>>> KIND,
>>>>> +EXPRESS OR
>>>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>>>> +MERCHANTABILITY,
>>>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
>>>> NO EVENT
>>>>> +SHALL
>>>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>>>> +DAMAGES OR
>>>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>>>> +OTHERWISE,
>>>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
>> OR
>>>> THE USE
>>>>> +OR
>>>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>>>> + */
>>>>> +
>>>>> +#ifndef __MXGPU_VI_H__
>>>>> +#define __MXGPU_VI_H__
>>>>> +
>>>>> +#define VI_MAILBOX_TIMEDOUT	150
>>>>> +#define VI_MAILBOX_RESET_TIME	12
>>>>> +
>>>>> +/* VI mailbox messages request */
>>>>> +enum idh_request {
>>>>> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
>>>>> +	IDH_REL_GPU_INIT_ACCESS,
>>>>> +	IDH_REQ_GPU_FINI_ACCESS,
>>>>> +	IDH_REL_GPU_FINI_ACCESS,
>>>>> +	IDH_REQ_GPU_RESET_ACCESS
>>>>> +};
>>>>> +
>>>>> +/* VI mailbox messages data */
>>>>> +enum idh_event {
>>>>> +	IDH_CLR_MSG_BUF = 0,
>>>>> +	IDH_READY_TO_ACCESS_GPU,
>>>>> +	IDH_FLR_NOTIFICATION,
>>>>> +	IDH_FLR_NOTIFICATION_CMPL,
>>>>> +	IDH_EVENT_MAX
>>>>> +};
>>>>> +
>>>>> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
>>>>> +
>>>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
>>>>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
>>>>> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
>>>>> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
>>>>> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
>>>>> +
>>>>> +#endif
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

Am 10.01.2017 um 15:01 schrieb Yu, Xiangliang:
>> -----Original Message-----
>> From: Christian König [mailto:deathsimple@vodafone.de]
>> Sent: Tuesday, January 10, 2017 9:09 PM
>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
>> gfx@lists.freedesktop.org
>> Subject: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
>>
>> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
>>> For virtualization, it is must for driver to use KIQ to access
>>> registers when it is out of GPU full access mode.
>>>
>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86
>> ++++++++++++++++++++++++++++++
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>>>    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>>>    5 files changed, 101 insertions(+), 1 deletion(-)
>>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 4185b03..0b8e470 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>>    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>>>    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>>>    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
>>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
>>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>>>
>>>    # add asic specific block
>>>    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
>> kv_dpm.o \
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index f82919d..9a2fd3e 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device
>> *adev, uint32_t reg,
>>>    {
>>>    	uint32_t ret;
>>>
>>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
>>> +		return amdgpu_virt_kiq_rreg(adev, reg);
>>> +
>>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>>>    	else {
>>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device
>> *adev, uint32_t reg, uint32_t v,
>>>    {
>>>    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>>>
>>> +	if (amdgpu_sriov_runtime(adev))
>>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
>>> +
>>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>>>    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>>>    	else {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> new file mode 100644
>>> index 0000000..6520a4e
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>>> @@ -0,0 +1,86 @@
>>> +/*
>>> + * Copyright 2017 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person
>>> +obtaining a
>>> + * copy of this software and associated documentation files (the
>>> +"Software"),
>>> + * to deal in the Software without restriction, including without
>>> +limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute,
>>> +sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom
>>> +the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be
>>> +included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>> KIND,
>>> +EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
>>> +MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
>> NO EVENT
>>> +SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
>>> +DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
>>> +OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
>> THE USE
>>> +OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +#include "amdgpu_virt.h"
>>> +
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
>>> +	mutex_init(&adev->virt.lock);
>>> +}
>>> +
>>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
>>> +reg) {
>>> +	signed long r;
>>> +	uint32_t val;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_rreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_rreg(ring, reg);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	fence_get(f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	fence_put(f);
>> Why do you grab and release an extra fence reference here?
> KIQ ring has no scheduler, I think we need to take care fence.

No, that is nonsense. Fence reference counting is completely separate to 
the scheduler.

Please just drop the extra fence_get()/fence_put() dance here.

amdgpu_fence_emit() returns the fence with one reference held which you 
need to drop before "f" leaves scope here. No need for anything else.

Regards,
Christian.

>
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +
>>> +	val = adev->wb.wb[adev->virt.val_offs];
>>> +
>>> +	return val;
>>> +}
>>> +
>>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v) {
>>> +	signed long r;
>>> +	struct fence *f;
>>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
>>> +	struct amdgpu_ring *ring = &kiq->ring;
>>> +
>>> +	BUG_ON(!ring->funcs->emit_wreg);
>>> +
>>> +	mutex_lock(&adev->virt.lock);
>>> +	amdgpu_ring_alloc(ring, 32);
>>> +	amdgpu_ring_emit_hdp_flush(ring);
>>> +	amdgpu_ring_emit_wreg(ring, reg, v);
>>> +	amdgpu_ring_emit_hdp_invalidate(ring);
>>> +	amdgpu_fence_emit(ring, &f);
>>> +	fence_get(f);
>>> +	amdgpu_ring_commit(ring);
>>> +	mutex_unlock(&adev->virt.lock);
>>> +
>>> +	r = fence_wait(f, false);
>>> +	fence_put(f);
>>> +	if (r)
>>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>>> +	fence_put(f);
>>> +}
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> index 79619b6..24f0590 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
>>> @@ -33,6 +33,7 @@
>>>    struct amdgpu_virt {
>>>    	uint32_t		caps;
>>>    	uint32_t		val_offs;
>>> +	struct mutex		lock;
>>>    };
>>>
>>>    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline
>>> bool is_virtual_machine(void)
>>>    #endif
>>>    }
>>>
>>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
>>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
>>> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
>>> +uint32_t v);
>>> +
>>>    #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
>>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>>>    		(amdgpu_ip_block_mask & (1 <<
>> AMD_IP_BLOCK_TYPE_SMC)))
>>>    		smc_enabled = true;
>>>
>>> +	if (amdgpu_sriov_vf(adev))
>>> +		amdgpu_virt_init_setting(adev);
>>> +
>>>    	adev->rev_id = vi_get_rev_id(adev);
>>>    	adev->external_rev_id = 0xFF;
>>>    	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 10:08 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
> 
> Am 10.01.2017 um 15:01 schrieb Yu, Xiangliang:
> >> -----Original Message-----
> >> From: Christian König [mailto:deathsimple@vodafone.de]
> >> Sent: Tuesday, January 10, 2017 9:09 PM
> >> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> >> gfx@lists.freedesktop.org
> >> Subject: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
> >>
> >> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> >>> For virtualization, it is must for driver to use KIQ to access
> >>> registers when it is out of GPU full access mode.
> >>>
> >>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86
> >> ++++++++++++++++++++++++++++++
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
> >>>    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
> >>>    5 files changed, 101 insertions(+), 1 deletion(-)
> >>>    create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> index 4185b03..0b8e470 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
> >>>    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
> >>>    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
> >>>    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> >>> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> >>> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
> >>>
> >>>    # add asic specific block
> >>>    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o
> >> kv_dpm.o \
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> index f82919d..9a2fd3e 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> >>> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device
> >> *adev, uint32_t reg,
> >>>    {
> >>>    	uint32_t ret;
> >>>
> >>> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> >>> +		return amdgpu_virt_kiq_rreg(adev, reg);
> >>> +
> >>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >>>    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
> >>>    	else {
> >>> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device
> >> *adev, uint32_t reg, uint32_t v,
> >>>    {
> >>>    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
> >>>
> >>> +	if (amdgpu_sriov_runtime(adev))
> >>> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> >>> +
> >>>    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >>>    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
> >>>    	else {
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> new file mode 100644
> >>> index 0000000..6520a4e
> >>> --- /dev/null
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> @@ -0,0 +1,86 @@
> >>> +/*
> >>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>> + *
> >>> + * Permission is hereby granted, free of charge, to any person
> >>> +obtaining a
> >>> + * copy of this software and associated documentation files (the
> >>> +"Software"),
> >>> + * to deal in the Software without restriction, including without
> >>> +limitation
> >>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>> +sublicense,
> >>> + * and/or sell copies of the Software, and to permit persons to
> >>> +whom the
> >>> + * Software is furnished to do so, subject to the following conditions:
> >>> + *
> >>> + * The above copyright notice and this permission notice shall be
> >>> +included in
> >>> + * all copies or substantial portions of the Software.
> >>> + *
> >>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> >> KIND,
> >>> +EXPRESS OR
> >>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>> +MERCHANTABILITY,
> >>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> >> NO EVENT
> >>> +SHALL
> >>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> >>> +DAMAGES OR
> >>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> >>> +OTHERWISE,
> >>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> OR
> >> THE USE
> >>> +OR
> >>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>> + */
> >>> +
> >>> +#include "amdgpu.h"
> >>> +#include "amdgpu_virt.h"
> >>> +
> >>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> >>> +	mutex_init(&adev->virt.lock);
> >>> +}
> >>> +
> >>> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> >>> +reg) {
> >>> +	signed long r;
> >>> +	uint32_t val;
> >>> +	struct fence *f;
> >>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> >>> +	struct amdgpu_ring *ring = &kiq->ring;
> >>> +
> >>> +	BUG_ON(!ring->funcs->emit_rreg);
> >>> +
> >>> +	mutex_lock(&adev->virt.lock);
> >>> +	amdgpu_ring_alloc(ring, 32);
> >>> +	amdgpu_ring_emit_hdp_flush(ring);
> >>> +	amdgpu_ring_emit_rreg(ring, reg);
> >>> +	amdgpu_ring_emit_hdp_invalidate(ring);
> >>> +	amdgpu_fence_emit(ring, &f);
> >>> +	fence_get(f);
> >>> +	amdgpu_ring_commit(ring);
> >>> +	mutex_unlock(&adev->virt.lock);
> >>> +
> >>> +	r = fence_wait(f, false);
> >>> +	fence_put(f);
> >> Why do you grab and release an extra fence reference here?
> > KIQ ring has no scheduler, I think we need to take care fence.
> 
> No, that is nonsense. Fence reference counting is completely separate to the
> scheduler.
> 
> Please just drop the extra fence_get()/fence_put() dance here.
> 
> amdgpu_fence_emit() returns the fence with one reference held which you
> need to drop before "f" leaves scope here. No need for anything else.

Let me check with Monk.

> 
> >
> >>> +	if (r)
> >>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >>> +	fence_put(f);
> >>> +
> >>> +	val = adev->wb.wb[adev->virt.val_offs];
> >>> +
> >>> +	return val;
> >>> +}
> >>> +
> >>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> >>> +uint32_t v) {
> >>> +	signed long r;
> >>> +	struct fence *f;
> >>> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> >>> +	struct amdgpu_ring *ring = &kiq->ring;
> >>> +
> >>> +	BUG_ON(!ring->funcs->emit_wreg);
> >>> +
> >>> +	mutex_lock(&adev->virt.lock);
> >>> +	amdgpu_ring_alloc(ring, 32);
> >>> +	amdgpu_ring_emit_hdp_flush(ring);
> >>> +	amdgpu_ring_emit_wreg(ring, reg, v);
> >>> +	amdgpu_ring_emit_hdp_invalidate(ring);
> >>> +	amdgpu_fence_emit(ring, &f);
> >>> +	fence_get(f);
> >>> +	amdgpu_ring_commit(ring);
> >>> +	mutex_unlock(&adev->virt.lock);
> >>> +
> >>> +	r = fence_wait(f, false);
> >>> +	fence_put(f);
> >>> +	if (r)
> >>> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >>> +	fence_put(f);
> >>> +}
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> index 79619b6..24f0590 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> @@ -33,6 +33,7 @@
> >>>    struct amdgpu_virt {
> >>>    	uint32_t		caps;
> >>>    	uint32_t		val_offs;
> >>> +	struct mutex		lock;
> >>>    };
> >>>
> >>>    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static
> >>> inline bool is_virtual_machine(void)
> >>>    #endif
> >>>    }
> >>>
> >>> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
> >>> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
> >>> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> >>> +uint32_t v);
> >>> +
> >>>    #endif
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> >>> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
> >>>    		(amdgpu_ip_block_mask & (1 <<
> >> AMD_IP_BLOCK_TYPE_SMC)))
> >>>    		smc_enabled = true;
> >>>
> >>> +	if (amdgpu_sriov_vf(adev))
> >>> +		amdgpu_virt_init_setting(adev);
> >>> +
> >>>    	adev->rev_id = vi_get_rev_id(adev);
> >>>    	adev->external_rev_id = 0xFF;
> >>>    	switch (adev->asic_type) {
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 10:06 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
> interfaces
> 
> Am 10.01.2017 um 14:43 schrieb Yu, Xiangliang:
> >> -----Original Message-----
> >> From: Christian König [mailto:deathsimple@vodafone.de]
> >> Sent: Tuesday, January 10, 2017 9:30 PM
> >> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> >> gfx@lists.freedesktop.org
> >> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt operation
> >> interfaces
> >>
> >> Am 10.01.2017 um 14:20 schrieb Yu, Xiangliang:
> >>>> -----Original Message-----
> >>>> From: Christian König [mailto:deathsimple@vodafone.de]
> >>>> Sent: Tuesday, January 10, 2017 9:16 PM
> >>>> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> >>>> gfx@lists.freedesktop.org
> >>>> Subject: Re: [V2 06/11] drm/amdgpu/virt: implement VI virt
> >>>> operation interfaces
> >>>>
> >>>> Unrelated to the patch, but what editor and settings do you use?
> >>> Vim
> >> dito.
> >>
> >>>> See this code for example
> >>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> >>>>> +				 TRN_MSG_VALID, val ? 1 : 0);
> >>>> The indentation seems to be incorrect.
> >>> In my platform, there is no problem.
> >> Are you using linuxsty.vim or something else?
> > No, I configure it by myself.
> 
> Ah, there it comes from. Please use linuxsty.vim or any other predefined set
> of setting for vim.
> 
> Configuring it yourself is usually not a good idea, cause that can easily lead to
> slightly different coding styles.

All right

> 
> >
> >> If I'm not completely mistaken "TRN_MSG_VALID" should be on the same
> >> indentation as "reg" and not as "MAILBOX_CONTROL".
> > I think it should be on same as "MAILBOX_CONTROL" as it is also macro and
> capital.
> 
> No, function arguments broken to the next line should start at the same
> place as the "(" of the previous line.
> 
> See the other code in amdgpu as well or just use linuxsty.

Got it. I'll also check other code later

> 
> Regards,
> Christian.
> 
> >
> >>>> Not much of a problem, but if we add this like it is anybody else
> >>>> editing the file could mess up the indentation when the editor does
> that
> >> differently.
> >>>> Regards,
> >>>> Christian.
> >>>>
> >>>> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> >>>>> VI has asic specific virt support, which including mailbox and
> >>>>> golden registers init.
> >>>>>
> >>>>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> >>>>> ---
> >>>>>     drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
> >>>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
> >>>>>     drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> >>>> +++++++++++++++++++++++++++++++
> >>>>>     drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
> >>>>>     4 files changed, 651 insertions(+), 1 deletion(-)
> >>>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>>>>     create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>>>>
> >>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>>>> b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>>>> index 0b8e470..388e7b0 100644
> >>>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> >>>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> >>>>> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+=
> cik.o
> >>>> cik_ih.o kv_smc.o kv_dpm.o \
> >>>>>     amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o
> gfx_v6_0.o
> >>>> si_ih.o
> >>>>> si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> >>>>>
> >>>>>     amdgpu-y += \
> >>>>> -	vi.o
> >>>>> +	vi.o mxgpu_vi.o
> >>>>>
> >>>>>     # add GMC block
> >>>>>     amdgpu-y += \
> >>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>>>> index 3f8fc0f..611676b 100644
> >>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>>>> @@ -45,6 +45,9 @@ struct amdgpu_virt {
> >>>>>     	uint32_t		val_offs;
> >>>>>     	struct mutex		lock;
> >>>>>
> >>>>> +	struct amdgpu_irq_src		ack_irq;
> >>>>> +	struct amdgpu_irq_src		rcv_irq;
> >>>>> +	struct delayed_work		flr_work;
> >>>>>     	const struct amdgpu_virt_ops	*ops;
> >>>>>     };
> >>>>>
> >>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>>>> new file mode 100644
> >>>>> index 0000000..7d44663
> >>>>> --- /dev/null
> >>>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> >>>>> @@ -0,0 +1,592 @@
> >>>>> +/*
> >>>>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>>>> + *
> >>>>> + * Permission is hereby granted, free of charge, to any person
> >>>>> +obtaining a
> >>>>> + * copy of this software and associated documentation files (the
> >>>>> +"Software"),
> >>>>> + * to deal in the Software without restriction, including without
> >>>>> +limitation
> >>>>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>>>> +sublicense,
> >>>>> + * and/or sell copies of the Software, and to permit persons to
> >>>>> +whom the
> >>>>> + * Software is furnished to do so, subject to the following conditions:
> >>>>> + *
> >>>>> + * The above copyright notice and this permission notice shall be
> >>>>> +included in
> >>>>> + * all copies or substantial portions of the Software.
> >>>>> + *
> >>>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
> ANY
> >>>> KIND,
> >>>>> +EXPRESS OR
> >>>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>>>> +MERCHANTABILITY,
> >>>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
> IN
> >>>> NO EVENT
> >>>>> +SHALL
> >>>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY
> CLAIM,
> >>>>> +DAMAGES OR
> >>>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
> OR
> >>>>> +OTHERWISE,
> >>>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> >> OR
> >>>> THE USE
> >>>>> +OR
> >>>>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>>>> + *
> >>>>> + * Authors: Xiangliang.Yu@amd.com
> >>>>> + */
> >>>>> +
> >>>>> +#include "amdgpu.h"
> >>>>> +#include "vi.h"
> >>>>> +#include "bif/bif_5_0_d.h"
> >>>>> +#include "bif/bif_5_0_sh_mask.h"
> >>>>> +#include "vid.h"
> >>>>> +#include "gca/gfx_8_0_d.h"
> >>>>> +#include "gca/gfx_8_0_sh_mask.h"
> >>>>> +#include "gmc_v8_0.h"
> >>>>> +#include "gfx_v8_0.h"
> >>>>> +#include "sdma_v3_0.h"
> >>>>> +#include "tonga_ih.h"
> >>>>> +#include "gmc/gmc_8_2_d.h"
> >>>>> +#include "gmc/gmc_8_2_sh_mask.h"
> >>>>> +#include "oss/oss_3_0_d.h"
> >>>>> +#include "oss/oss_3_0_sh_mask.h"
> >>>>> +#include "gca/gfx_8_0_sh_mask.h"
> >>>>> +#include "dce/dce_10_0_d.h"
> >>>>> +#include "dce/dce_10_0_sh_mask.h"
> >>>>> +#include "smu/smu_7_1_3_d.h"
> >>>>> +#include "mxgpu_vi.h"
> >>>>> +
> >>>>> +/* VI golden setting */
> >>>>> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> >>>>> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> >>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>>>> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> >>>>> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> >>>>> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> >>>>> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> >>>>> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> >>>>> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> >>>>> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> >>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>>>> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> >>>>> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> >>>>> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> >>>>> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> >>>>> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> >>>>> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> >>>>> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> >>>>> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> >>>>> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> >>>>> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> >>>>> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> >>>>> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> >>>>> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> >>>>> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> >>>>> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
> >>>>> +
> >>>>> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> >>>>> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> >>>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> >>>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> >>>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> >>>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> >>>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> >>>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> >>>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> >>>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> >>>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> >>>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> >>>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> >>>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> >>>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> >>>>> +
> >>>>> +static const u32 xgpu_fiji_golden_common_all[] = {
> >>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>>>> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> >>>>> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> >>>>> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> >>>>> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> >>>>> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> >>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff,
> 0x00007FBF,
> >>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff,
> 0x00007FAF,
> >>>>> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> >>>>> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
> >>>>> +
> >>>>> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> >>>>> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> >>>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> >>>>> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> >>>>> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> >>>>> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> >>>>> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> >>>>> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> >>>>> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> >>>>> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> >>>>> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> >>>>> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> >>>>> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> >>>>> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> >>>>> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> >>>>> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> >>>>> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> >>>>> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> >>>>> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> >>>>> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> >>>>> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> >>>>> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> >>>>> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> >>>>> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> >>>>> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> >>>>> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> >>>>> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> >>>>> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> >>>>> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> >>>>> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> >>>>> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> >>>>> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> >>>>> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> >>>>> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> >>>>> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> >>>>> +};
> >>>>> +
> >>>>> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> >>>>> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> >>>>> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> >>>>> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> >>>>> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> >>>>> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> >>>>> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> >>>>> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> >>>>> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> >>>>> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> >>>>> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> >>>>> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> >>>>> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> >>>>> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> >>>>> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> >>>>> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> >>>>> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>>>> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> >>>>> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> >>>>> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> >>>>> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> >>>>> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> >>>>> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> >>>>> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> >>>>> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> >>>>> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> >>>>> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> >>>>> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> >>>>> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> >>>>> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> >>>>> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
> >>>>> +
> >>>>> +static const u32 xgpu_tonga_golden_common_all[] = {
> >>>>> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> >>>>> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> >>>>> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> >>>>> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> >>>>> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> >>>>> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> >>>>> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff,
> 0x00007FBF, };
> >>>>> +
> >>>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
> >>>>> +	switch (adev->asic_type) {
> >>>>> +	case CHIP_FIJI:
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> xgpu_fiji_mgcg_cgcg_init,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> xgpu_fiji_mgcg_cgcg_init));
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> >>>> xgpu_fiji_golden_settings_a10,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> >>>> xgpu_fiji_golden_settings_a10));
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> >>>> xgpu_fiji_golden_common_all,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> >>>> xgpu_fiji_golden_common_all));
> >>>>> +		break;
> >>>>> +	case CHIP_TONGA:
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> xgpu_tonga_mgcg_cgcg_init,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> >>>> xgpu_tonga_mgcg_cgcg_init));
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> >>>> xgpu_tonga_golden_settings_a11,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> >>>> xgpu_tonga_golden_settings_a11));
> >>>>> +		amdgpu_program_register_sequence(adev,
> >>>>> +
> >>>> xgpu_tonga_golden_common_all,
> >>>>> +						 (const
> u32)ARRAY_SIZE(
> >>>>> +
> >>>> xgpu_tonga_golden_common_all));
> >>>>> +		break;
> >>>>> +	default:
> >>>>> +		BUG_ON("Doesn't support chip type.\n");
> >>>>> +		break;
> >>>>> +	}
> >>>>> +}
> >>>>> +
> >>>>> +/*
> >>>>> + * Mailbox communication between GPU hypervisor and VFs  */
> static
> >>>>> +void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
> >>>>> +	u32 reg;
> >>>>> +
> >>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> RCV_MSG_ACK, 1);
> >>>>> +	WREG32(mmMAILBOX_CONTROL, reg);
> >>>>> +}
> >>>>> +
> >>>>> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
> >>>>> +bool val) {
> >>>>> +	u32 reg;
> >>>>> +
> >>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> >>>>> +				 TRN_MSG_VALID, val ? 1 : 0);
> >>>>> +	WREG32(mmMAILBOX_CONTROL, reg);
> >>>>> +}
> >>>>> +
> >>>>> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device
> *adev,
> >>>>> +				      enum idh_event event)
> >>>>> +{
> >>>>> +	u32 reg;
> >>>>> +
> >>>>> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> >>>>> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> >>>>> +				 MSGBUF_DATA, event);
> >>>>> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> >>>>> +
> >>>>> +	xgpu_vi_mailbox_set_valid(adev, true); }
> >>>>> +
> >>>>> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> >>>>> +				   enum idh_event event)
> >>>>> +{
> >>>>> +	u32 reg;
> >>>>> +
> >>>>> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> >>>>> +	if (reg != event)
> >>>>> +		return -ENOENT;
> >>>>> +
> >>>>> +	/* send ack to PF */
> >>>>> +	xgpu_vi_mailbox_send_ack(adev);
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
> >>>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> >>>>> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL,
> TRN_MSG_ACK);
> >>>>> +	u32 reg;
> >>>>> +
> >>>>> +	reg = RREG32(mmMAILBOX_CONTROL);
> >>>>> +	while (!(reg & mask)) {
> >>>>> +		if (timeout <= 0) {
> >>>>> +			pr_err("Doesn't get ack from pf.\n");
> >>>>> +			r = -ETIME;
> >>>>> +			break;
> >>>>> +		}
> >>>>> +		msleep(1);
> >>>>> +		timeout -= 1;
> >>>>> +
> >>>>> +		reg = RREG32(mmMAILBOX_CONTROL);
> >>>>> +	}
> >>>>> +
> >>>>> +	return r;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
> >>>>> +idh_event event) {
> >>>>> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> >>>>> +
> >>>>> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> >>>>> +	while (r) {
> >>>>> +		if (timeout <= 0) {
> >>>>> +			pr_err("Doesn't get ack from pf.\n");
> >>>>> +			r = -ETIME;
> >>>>> +			break;
> >>>>> +		}
> >>>>> +		msleep(1);
> >>>>> +		timeout -= 1;
> >>>>> +
> >>>>> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> >>>>> +	}
> >>>>> +
> >>>>> +	return r;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_send_access_requests(struct amdgpu_device
> *adev,
> >>>>> +					enum idh_request request)
> >>>>> +{
> >>>>> +	int r;
> >>>>> +
> >>>>> +	xgpu_vi_mailbox_trans_msg(adev, request);
> >>>>> +
> >>>>> +	/* start to poll ack */
> >>>>> +	r = xgpu_vi_poll_ack(adev);
> >>>>> +	if (r)
> >>>>> +		return r;
> >>>>> +
> >>>>> +	xgpu_vi_mailbox_set_valid(adev, false);
> >>>>> +
> >>>>> +	/* start to check msg if request is idh_req_gpu_init_access */
> >>>>> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> >>>>> +		r = xgpu_vi_poll_msg(adev,
> IDH_READY_TO_ACCESS_GPU);
> >>>>> +		if (r)
> >>>>> +			return r;
> >>>>> +	}
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
> >>>>> +	return xgpu_vi_send_access_requests(adev,
> >>>> IDH_REQ_GPU_RESET_ACCESS);
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device
> >> *adev,
> >>>>> +					   bool init)
> >>>>> +{
> >>>>> +	enum idh_event event;
> >>>>> +
> >>>>> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> >>>> IDH_REQ_GPU_FINI_ACCESS;
> >>>>> +	return xgpu_vi_send_access_requests(adev, event); }
> >>>>> +
> >>>>> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device
> >> *adev,
> >>>>> +					   bool init)
> >>>>> +{
> >>>>> +	enum idh_event event;
> >>>>> +	int r = 0;
> >>>>> +
> >>>>> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> >>>> IDH_REL_GPU_FINI_ACCESS;
> >>>>> +	r = xgpu_vi_send_access_requests(adev, event);
> >>>>> +
> >>>>> +	return r;
> >>>>> +}
> >>>>> +
> >>>>> +/* add support mailbox interrupts */ static int
> >>>>> +xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> >>>>> +				   struct amdgpu_irq_src *source,
> >>>>> +				   struct amdgpu_iv_entry *entry) {
> >>>>> +	DRM_DEBUG("get ack intr and do nothing.\n");
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> >>>>> +				       struct amdgpu_irq_src *src,
> >>>>> +				       unsigned type,
> >>>>> +				       enum amdgpu_interrupt_state
> state) {
> >>>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> >>>>> +
> >>>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL,
> ACK_INT_EN,
> >>>>> +				 (state ==
> AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> >>>> 0);
> >>>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
> >>>>> +	struct amdgpu_virt *virt = container_of(work,
> >>>>> +					struct amdgpu_virt,
> flr_work.work);
> >>>>> +	struct amdgpu_device *adev = container_of(virt,
> >>>>> +					struct amdgpu_device, virt);
> >>>>> +	int r = 0;
> >>>>> +
> >>>>> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> >>>>> +	if (r)
> >>>>> +		DRM_ERROR("failed to get flr cmpl msg from
> hypervior.\n");
> >>>>> +
> >>>>> +	/* TODO: need to restore gfx states */ }
> >>>>> +
> >>>>> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> >>>>> +				       struct amdgpu_irq_src *src,
> >>>>> +				       unsigned type,
> >>>>> +				       enum amdgpu_interrupt_state
> state) {
> >>>>> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> >>>>> +
> >>>>> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL,
> VALID_INT_EN,
> >>>>> +				 (state ==
> AMDGPU_IRQ_STATE_ENABLE) ? 1 :
> >>>> 0);
> >>>>> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> >>>>> +				   struct amdgpu_irq_src *source,
> >>>>> +				   struct amdgpu_iv_entry *entry) {
> >>>>> +	int r;
> >>>>> +
> >>>>> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> >>>>> +	r = xgpu_vi_mailbox_rcv_msg(adev,
> IDH_FLR_NOTIFICATION);
> >>>>> +	/* do nothing for other msg */
> >>>>> +	if (r)
> >>>>> +		return 0;
> >>>>> +
> >>>>> +	/* TODO: need to save gfx states */
> >>>>> +	schedule_delayed_work(&adev->virt.flr_work,
> >>>>> +
> msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +static const struct amdgpu_irq_src_funcs
> >>>>> +xgpu_vi_mailbox_ack_irq_funcs
> >>>> = {
> >>>>> +	.set = xgpu_vi_set_mailbox_ack_irq,
> >>>>> +	.process = xgpu_vi_mailbox_ack_irq, };
> >>>>> +
> >>>>> +static const struct amdgpu_irq_src_funcs
> >>>>> +xgpu_vi_mailbox_rcv_irq_funcs
> >>>> = {
> >>>>> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> >>>>> +	.process = xgpu_vi_mailbox_rcv_irq, };
> >>>>> +
> >>>>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
> {
> >>>>> +	adev->virt.ack_irq.num_types = 1;
> >>>>> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> >>>>> +	adev->virt.rcv_irq.num_types = 1;
> >>>>> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
> >>>>> +
> >>>>> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
> >>>>> +	int r;
> >>>>> +
> >>>>> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> >>>>> +	if (r)
> >>>>> +		return r;
> >>>>> +
> >>>>> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> >>>>> +	if (r) {
> >>>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> >>>>> +		return r;
> >>>>> +	}
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
> >>>>> +	int r;
> >>>>> +
> >>>>> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> >>>>> +	if (r)
> >>>>> +		return r;
> >>>>> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> >>>>> +	if (r) {
> >>>>> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> >>>>> +		return r;
> >>>>> +	}
> >>>>> +
> >>>>> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> >>>> xgpu_vi_mailbox_flr_work);
> >>>>> +
> >>>>> +	return 0;
> >>>>> +}
> >>>>> +
> >>>>> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
> >>>>> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> >>>>> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> >>>>> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
> >>>>> +
> >>>>> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> >>>>> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> >>>>> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> >>>>> +	.reset_gpu		= xgpu_vi_request_reset,
> >>>>> +};
> >>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>>>> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>>>> new file mode 100644
> >>>>> index 0000000..fd6216e
> >>>>> --- /dev/null
> >>>>> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> >>>>> @@ -0,0 +1,55 @@
> >>>>> +/*
> >>>>> + * Copyright 2017 Advanced Micro Devices, Inc.
> >>>>> + *
> >>>>> + * Permission is hereby granted, free of charge, to any person
> >>>>> +obtaining a
> >>>>> + * copy of this software and associated documentation files (the
> >>>>> +"Software"),
> >>>>> + * to deal in the Software without restriction, including without
> >>>>> +limitation
> >>>>> + * the rights to use, copy, modify, merge, publish, distribute,
> >>>>> +sublicense,
> >>>>> + * and/or sell copies of the Software, and to permit persons to
> >>>>> +whom the
> >>>>> + * Software is furnished to do so, subject to the following conditions:
> >>>>> + *
> >>>>> + * The above copyright notice and this permission notice shall be
> >>>>> +included in
> >>>>> + * all copies or substantial portions of the Software.
> >>>>> + *
> >>>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
> ANY
> >>>> KIND,
> >>>>> +EXPRESS OR
> >>>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> >>>>> +MERCHANTABILITY,
> >>>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
> IN
> >>>> NO EVENT
> >>>>> +SHALL
> >>>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY
> CLAIM,
> >>>>> +DAMAGES OR
> >>>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
> OR
> >>>>> +OTHERWISE,
> >>>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> >> OR
> >>>> THE USE
> >>>>> +OR
> >>>>> + * OTHER DEALINGS IN THE SOFTWARE.
> >>>>> + */
> >>>>> +
> >>>>> +#ifndef __MXGPU_VI_H__
> >>>>> +#define __MXGPU_VI_H__
> >>>>> +
> >>>>> +#define VI_MAILBOX_TIMEDOUT	150
> >>>>> +#define VI_MAILBOX_RESET_TIME	12
> >>>>> +
> >>>>> +/* VI mailbox messages request */
> >>>>> +enum idh_request {
> >>>>> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> >>>>> +	IDH_REL_GPU_INIT_ACCESS,
> >>>>> +	IDH_REQ_GPU_FINI_ACCESS,
> >>>>> +	IDH_REL_GPU_FINI_ACCESS,
> >>>>> +	IDH_REQ_GPU_RESET_ACCESS
> >>>>> +};
> >>>>> +
> >>>>> +/* VI mailbox messages data */
> >>>>> +enum idh_event {
> >>>>> +	IDH_CLR_MSG_BUF = 0,
> >>>>> +	IDH_READY_TO_ACCESS_GPU,
> >>>>> +	IDH_FLR_NOTIFICATION,
> >>>>> +	IDH_FLR_NOTIFICATION_CMPL,
> >>>>> +	IDH_EVENT_MAX
> >>>>> +};
> >>>>> +
> >>>>> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> >>>>> +
> >>>>> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
> >>>>> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
> int
> >>>>> +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
> >>>>> +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
> >>>>> +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> >>>>> +
> >>>>> +#endif
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Yu, Xiangliang]

> -----Original Message-----
> From: Christian König [mailto:deathsimple@vodafone.de]
> Sent: Tuesday, January 10, 2017 9:59 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> Am 10.01.2017 um 14:33 schrieb Yu, Xiangliang:
> >> -----Original Message-----
> >> From: Christian König [mailto:deathsimple@vodafone.de]
> >> Sent: Tuesday, January 10, 2017 9:12 PM
> >> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> >> gfx@lists.freedesktop.org
> >> Subject: Re: [V2 05/11] drm/amdgpu/virt: add high level interfaces
> >> for virt
> >>
> >> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> >>> Add high level interfaces that is not relate to specific asic. So
> >>> asic files just need to implement the interfaces to support virtualization.
> >>>
> >>> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> >> ++++++++++++++++++++++++++++++++
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
> >>>    2 files changed, 72 insertions(+)
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> index 6520a4e..f32a789 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >>> @@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct
> amdgpu_device
> >> *adev, uint32_t reg, uint32_t v)
> >>>    		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> >>>    	fence_put(f);
> >>>    }
> >>> +
> >>> +/**
> >>> + * amdgpu_virt_request_full_gpu() - request full gpu access
> >>> + * @amdgpu:	amdgpu device.
> >>> + * @init:	is driver init time.
> >>> + * When start to init/fini driver, first need to request full gpu access.
> >>> + * Return: Zero if request success, otherwise will return error.
> >>> + */
> >>> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> >>> +init) {
> >>> +	struct amdgpu_virt *virt = &adev->virt;
> >>> +
> >>> +	if (virt->ops && virt->ops->req_full_gpu) {
> >>> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> >>> +		return virt->ops->req_full_gpu(adev, init);
> >> I would be conservative here and request full GPU access first and
> >> then clear AMDGPU_SRIOV_CAPS_RUNTIME.
> >>
> >> Just in case the function is called concurrently with another thread
> >> checking the caps.
> > It can't be used in parallel, the problem you said shouldn't be appear.
> >
> >> On the other hand req_full_gpu() could need the flag to handle
> >> register reads/writes correctly, but in this case I would question if
> >> we shouldn't add special macros for this.
> > We must clear RUNTIME flag so that can read/write registers with mmio,
> > Otherwise driver will hang.
> 
> Yeah, that's what I expected. Would be nice if we could better split that, e.g.
> have explicit macros for direct register write.
> 
> But that's really only nice to have, patch is ok as it is as far as I can see.

I think it is also very clear. Actually, I have a try but it is not good choice.
If anyone have good idea, can change it later.

> >
> >>
> >>
> >>> +	}
> >>> +
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +/**
> >>> + * amdgpu_virt_release_full_gpu() - release full gpu access
> >>> + * @amdgpu:	amdgpu device.
> >>> + * @init:	is driver init time.
> >>> + * When finishing driver init/fini, need to release full gpu access.
> >>> + * Return: Zero if release success, otherwise will returen error.
> >>> + */
> >>> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool
> >>> +init) {
> >>> +	struct amdgpu_virt *virt = &adev->virt;
> >>> +	int r;
> >>> +
> >>> +	if (virt->ops && virt->ops->rel_full_gpu) {
> >>> +		r = virt->ops->rel_full_gpu(adev, init);
> >>> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> >>> +		return r;
> >>> +	}
> >>> +	return 0;
> >>> +}
> >>> +
> >>> +/**
> >>> + * amdgpu_virt_reset_gpu() - reset gpu
> >>> + * @amdgpu:	amdgpu device.
> >>> + * Send reset command to GPU hypervisor to reset GPU that VM is
> >>> +using
> >>> + * Return: Zero if reset success, otherwise will return error.
> >>> + */
> >>> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
> >>> +	struct amdgpu_virt *virt = &adev->virt;
> >>> +
> >>> +	if (virt->ops && virt->ops->reset_gpu) {
> >>> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> >>> +		return virt->ops->reset_gpu(adev);
> >>> +	}
> >>> +
> >>> +	return 0;
> >>> +}
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> index 24f0590..3f8fc0f 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> >>> @@ -29,11 +29,23 @@
> >>>    #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a
> virtual
> >> function */
> >>>    #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole
> GPU
> >> is pass through for VM */
> >>>    #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full
> >> access mode */
> >>> +
> >>> +/**
> >>> + * struct amdgpu_virt_ops - amdgpu device virt operations  */
> >>> +struct amdgpu_virt_ops {
> >>> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> >>> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> >>> +	int (*reset_gpu)(struct amdgpu_device *adev); };
> >>> +
> >>>    /* GPU virtualization */
> >>>    struct amdgpu_virt {
> >>>    	uint32_t		caps;
> >>>    	uint32_t		val_offs;
> >>>    	struct mutex		lock;
> >>> +
> >>> +	const struct amdgpu_virt_ops	*ops;
> >>>    };
> >>>
> >>>    #define amdgpu_sriov_enabled(adev) \ @@ -63,5 +75,8 @@ static
> >>> inline bool is_virtual_machine(void)
> >>>    void amdgpu_virt_init_setting(struct amdgpu_device *adev);
> >>>    uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev,
> >>> uint32_t
> >> reg);
> >>>    void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
> >>> reg, uint32_t v);
> >>> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
> >>> +init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
> >>> +bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> >>>
> >>>    #endif
> 

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Yu, Xiangliang
> Sent: Tuesday, January 10, 2017 8:36 AM
> To: Christian König; amd-gfx@lists.freedesktop.org
> Cc: Liu, Monk
> Subject: RE: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization
> 
> > -----Original Message-----
> > From: Christian König [mailto:deathsimple@vodafone.de]
> > Sent: Tuesday, January 10, 2017 9:19 PM
> > To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> > gfx@lists.freedesktop.org
> > Cc: Liu, Monk <Monk.Liu@amd.com>
> > Subject: Re: [V2 11/11] drm/amdgpu: do not reset gpu for virtualization
> >
> > Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > > Current job timeout setting is not fit for virtualization, so it will
> > > cause job timeout sometimes. Add workaround for this when timeout
> > > happen.
> > >
> > > Signed-off-by: Monk Liu <Monk.Liu@amd.com>
> > > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > > ---
> > >   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
> > >   1 file changed, 3 insertions(+)
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > > b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > > index a6d828c..5705fbd 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > > @@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device
> > *adev)
> > >   	int resched;
> > >   	bool need_full_reset;
> > >
> > > +	if (amdgpu_sriov_vf(adev))
> >
> > At bare minimum we should print a warning here and increase or
> completely
> > disable the job timeout when virtualization is enabled.
> 
> The timeout function has the warning message, I think don't need more.
> The patch is just a workaround, later need to enhance the whole gpu reset
> process.

Please add a comment that explains that this is a temporary workaround, with that:
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 01/11] drm/amdgpu/ring: add two interfaces to support r/w registers with kiq

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 01/11] drm/amdgpu/ring: add two interfaces to support r/w
> registers with kiq
> 
> During virtual runtime, need to send command to kiq ring to
> read/write GPU registers. Add two interface to support the two
> actions.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h      | 2 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 ++
>  2 files changed, 4 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 13c30ab..13d87d7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -1646,6 +1646,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring
> *ring)
>  #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs-
> >emit_hdp_invalidate((r))
>  #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs-
> >emit_switch_buffer((r))
>  #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
> +#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
> +#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d),
> (v))
>  #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
>  #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
>  #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs-
> >patch_cond_exec((r),(o))
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index 92bc89b..c813cbe 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -135,6 +135,8 @@ struct amdgpu_ring_funcs {
>  	void (*end_use)(struct amdgpu_ring *ring);
>  	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
>  	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
> +	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
> +	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t
> val);
>  };
> 
>  struct amdgpu_ring {
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg function

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg function
> 
> Implement emit_rreg/wreg function for kiq ring.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  3 ++-
>  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 37
> ++++++++++++++++++++++++++++++++
>  2 files changed, 39 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 0d821d9..a6524bb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -30,7 +30,8 @@
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
>  /* GPU virtualization */
>  struct amdgpu_virt {
> -	uint32_t caps;
> +	uint32_t		caps;
> +	uint32_t		val_offs;

Maybe rename val_offs to register_val_offs to make it clearer what it is used for.  With that change, the patch is:
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>


>  };
> 
>  #define amdgpu_sriov_enabled(adev) \
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> index b487b97..e619f49 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> @@ -1373,6 +1373,12 @@ static int gfx_v8_0_kiq_init_ring(struct
> amdgpu_device *adev,
>  {
>  	int r = 0;
> 
> +	if (amdgpu_sriov_vf(adev)) {
> +		r = amdgpu_wb_get(adev, &adev->virt.val_offs);
> +		if (r)
> +			return r;
> +	}
> +
>  	ring->adev = NULL;
>  	ring->ring_obj = NULL;
>  	ring->use_doorbell = true;
> @@ -1399,6 +1405,9 @@ static int gfx_v8_0_kiq_init_ring(struct
> amdgpu_device *adev,
>  static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
>  				   struct amdgpu_irq_src *irq)
>  {
> +	if (amdgpu_sriov_vf(ring->adev))
> +		amdgpu_wb_free(ring->adev, ring->adev->virt.val_offs);
> +
>  	amdgpu_ring_fini(ring);
>  	irq->data = NULL;
>  }
> @@ -6681,6 +6690,32 @@ static void gfx_v8_ring_emit_cntxcntl(struct
> amdgpu_ring *ring, uint32_t flags)
>  	amdgpu_ring_write(ring, 0);
>  }
> 
> +static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t
> reg)
> +{
> +	struct amdgpu_device *adev = ring->adev;
> +
> +	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
> +	amdgpu_ring_write(ring, 0 |	/* src: register*/
> +				(5 << 8) |	/* dst: memory */
> +				(1 << 20));	/* write confirm */
> +	amdgpu_ring_write(ring, reg);
> +	amdgpu_ring_write(ring, 0);
> +	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
> +				adev->virt.val_offs * 4));
> +	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
> +				adev->virt.val_offs * 4));
> +}
> +
> +static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t
> reg,
> +				  uint32_t val)
> +{
> +	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
> +	amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
> +	amdgpu_ring_write(ring, reg);
> +	amdgpu_ring_write(ring, 0);
> +	amdgpu_ring_write(ring, val);
> +}
> +
>  static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device
> *adev,
>  						 enum
> amdgpu_interrupt_state state)
>  {
> @@ -6995,6 +7030,8 @@ static const struct amdgpu_ring_funcs
> gfx_v8_0_ring_funcs_kiq = {
>  	.test_ib = gfx_v8_0_ring_test_ib,
>  	.insert_nop = amdgpu_ring_insert_nop,
>  	.pad_ib = amdgpu_ring_generic_pad_ib,
> +	.emit_rreg = gfx_v8_0_ring_emit_rreg,
> +	.emit_wreg = gfx_v8_0_ring_emit_wreg,
>  };
> 
>  static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 03/11] drm/amdgpu/virt: add runtime flag

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 03/11] drm/amdgpu/virt: add runtime flag
> 
> Add new flag to define gpu runtime that is out of full gpu access.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index a6524bb..79619b6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -28,6 +28,7 @@
>  #define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled
> on this GPU */
>  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
> +#define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access
> mode */
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t		caps;
> @@ -43,6 +44,9 @@ struct amdgpu_virt {
>  #define amdgpu_sriov_bios(adev) \
>  ((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
> 
> +#define amdgpu_sriov_runtime(adev) \
> +((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME)
> +
>  #define amdgpu_passthrough(adev) \
>  ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
> 
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 05/11] drm/amdgpu/virt: add high level interfaces for virt
> 
> Add high level interfaces that is not relate to specific asic. So
> asic files just need to implement the interfaces to support
> virtualization.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
> ++++++++++++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 15 +++++++++
>  2 files changed, 72 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index 6520a4e..f32a789 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -84,3 +84,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
> *adev, uint32_t reg, uint32_t v)
>  		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
>  	fence_put(f);
>  }
> +
> +/**
> + * amdgpu_virt_request_full_gpu() - request full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When start to init/fini driver, first need to request full gpu access.
> + * Return: Zero if request success, otherwise will return error.
> + */
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->req_full_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->req_full_gpu(adev, init);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_release_full_gpu() - release full gpu access
> + * @amdgpu:	amdgpu device.
> + * @init:	is driver init time.
> + * When finishing driver init/fini, need to release full gpu access.
> + * Return: Zero if release success, otherwise will returen error.
> + */
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +	int r;
> +
> +	if (virt->ops && virt->ops->rel_full_gpu) {
> +		r = virt->ops->rel_full_gpu(adev, init);
> +		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return r;
> +	}
> +	return 0;
> +}
> +
> +/**
> + * amdgpu_virt_reset_gpu() - reset gpu
> + * @amdgpu:	amdgpu device.
> + * Send reset command to GPU hypervisor to reset GPU that VM is using
> + * Return: Zero if reset success, otherwise will return error.
> + */
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_virt *virt = &adev->virt;
> +
> +	if (virt->ops && virt->ops->reset_gpu) {
> +		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +		return virt->ops->reset_gpu(adev);
> +	}
> +
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 24f0590..3f8fc0f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -29,11 +29,23 @@
>  #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
> function */
>  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is
> pass through for VM */
>  #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access
> mode */
> +
> +/**
> + * struct amdgpu_virt_ops - amdgpu device virt operations
> + */
> +struct amdgpu_virt_ops {
> +	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
> +	int (*reset_gpu)(struct amdgpu_device *adev);
> +};
> +
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t		caps;
>  	uint32_t		val_offs;
>  	struct mutex		lock;
> +
> +	const struct amdgpu_virt_ops	*ops;
>  };
> 
>  #define amdgpu_sriov_enabled(adev) \
> @@ -63,5 +75,8 @@ static inline bool is_virtual_machine(void)
>  void amdgpu_virt_init_setting(struct amdgpu_device *adev);
>  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
>  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> uint32_t v);
> +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
> +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
> +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
> 
>  #endif
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 06/11] drm/amdgpu/virt: implement VI virt operation interfaces
> 
> VI has asic specific virt support, which including mailbox and
> golden registers init.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   3 +
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
> +++++++++++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
>  4 files changed, 651 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
>  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 0b8e470..388e7b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o
> cik_ih.o kv_smc.o kv_dpm.o \
>  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o
> si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
> 
>  amdgpu-y += \
> -	vi.o
> +	vi.o mxgpu_vi.o
> 
>  # add GMC block
>  amdgpu-y += \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 3f8fc0f..611676b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -45,6 +45,9 @@ struct amdgpu_virt {
>  	uint32_t		val_offs;
>  	struct mutex		lock;
> 
> +	struct amdgpu_irq_src		ack_irq;
> +	struct amdgpu_irq_src		rcv_irq;
> +	struct delayed_work		flr_work;
>  	const struct amdgpu_virt_ops	*ops;
>  };
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> new file mode 100644
> index 0000000..7d44663
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
> @@ -0,0 +1,592 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + *
> + * Authors: Xiangliang.Yu@amd.com
> + */
> +
> +#include "amdgpu.h"
> +#include "vi.h"
> +#include "bif/bif_5_0_d.h"
> +#include "bif/bif_5_0_sh_mask.h"
> +#include "vid.h"
> +#include "gca/gfx_8_0_d.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "gmc_v8_0.h"
> +#include "gfx_v8_0.h"
> +#include "sdma_v3_0.h"
> +#include "tonga_ih.h"
> +#include "gmc/gmc_8_2_d.h"
> +#include "gmc/gmc_8_2_sh_mask.h"
> +#include "oss/oss_3_0_d.h"
> +#include "oss/oss_3_0_sh_mask.h"
> +#include "gca/gfx_8_0_sh_mask.h"
> +#include "dce/dce_10_0_d.h"
> +#include "dce/dce_10_0_sh_mask.h"
> +#include "smu/smu_7_1_3_d.h"
> +#include "mxgpu_vi.h"
> +
> +/* VI golden setting */
> +static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
> +	mmPCIE_DATA, 0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_fiji_golden_settings_a10[] = {
> +	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +};
> +
> +static const u32 xgpu_fiji_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
> +	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
> +	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
> +	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
> +	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
> +	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
> +};
> +
> +static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
> +	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
> +	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
> +	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
> +	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
> +	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
> +	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
> +	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
> +	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
> +	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
> +	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
> +	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
> +	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
> +	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
> +	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
> +	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
> +	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
> +	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
> +	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
> +	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
> +	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
> +	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
> +	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
> +	mmPCIE_DATA,                0x000f0000, 0x00000000,
> +	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
> +	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
> +	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
> +	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
> +	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
> +	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
> +	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
> +	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
> +	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
> +	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
> +};
> +
> +static const u32 xgpu_tonga_golden_settings_a11[] = {
> +	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
> +	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
> +	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
> +	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
> +	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
> +	mmFBC_MISC, 0x1f311fff, 0x12300000,
> +	mmGB_GPU_ID, 0x0000000f, 0x00000000,
> +	mmHDMI_CONTROL, 0x31000111, 0x00000011,
> +	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
> +	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
> +	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
> +	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
> +	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
> +	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
> +	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
> +	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
> +	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
> +	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
> +	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
> +	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
> +	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
> +	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
> +	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
> +	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
> +	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
> +	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
> +	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
> +};
> +
> +static const u32 xgpu_tonga_golden_common_all[] = {
> +	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
> +	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
> +	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
> +	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
> +	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
> +	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
> +};
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
> +{
> +	switch (adev->asic_type) {
> +	case CHIP_FIJI:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_fiji_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +						 xgpu_fiji_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_settings_a10,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_settings_a10));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_fiji_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_fiji_golden_common_all));
> +		break;
> +	case CHIP_TONGA:
> +		amdgpu_program_register_sequence(adev,
> +						 xgpu_tonga_mgcg_cgcg_init,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_mgcg_cgcg_init));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_settings_a11,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_settings_a11));
> +		amdgpu_program_register_sequence(adev,
> +
> xgpu_tonga_golden_common_all,
> +						 (const u32)ARRAY_SIZE(
> +
> xgpu_tonga_golden_common_all));
> +		break;
> +	default:
> +		BUG_ON("Doesn't support chip type.\n");
> +		break;
> +	}
> +}
> +
> +/*
> + * Mailbox communication between GPU hypervisor and VFs
> + */
> +static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool
> val)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
> +				 TRN_MSG_VALID, val ? 1 : 0);
> +	WREG32(mmMAILBOX_CONTROL, reg);
> +}
> +
> +static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
> +				      enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
> +	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
> +				 MSGBUF_DATA, event);
> +	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
> +
> +	xgpu_vi_mailbox_set_valid(adev, true);
> +}
> +
> +static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
> +				   enum idh_event event)
> +{
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
> +	if (reg != event)
> +		return -ENOENT;
> +
> +	/* send ack to PF */
> +	xgpu_vi_mailbox_send_ack(adev);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_poll_ack(struct amdgpu_device *adev)
> +{
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL,
> TRN_MSG_ACK);
> +	u32 reg;
> +
> +	reg = RREG32(mmMAILBOX_CONTROL);
> +	while (!(reg & mask)) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		reg = RREG32(mmMAILBOX_CONTROL);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event
> event)
> +{
> +	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
> +
> +	r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	while (r) {
> +		if (timeout <= 0) {
> +			pr_err("Doesn't get ack from pf.\n");
> +			r = -ETIME;
> +			break;
> +		}
> +		msleep(1);
> +		timeout -= 1;
> +
> +		r = xgpu_vi_mailbox_rcv_msg(adev, event);
> +	}
> +
> +	return r;
> +}
> +
> +static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
> +					enum idh_request request)
> +{
> +	int r;
> +
> +	xgpu_vi_mailbox_trans_msg(adev, request);
> +
> +	/* start to poll ack */
> +	r = xgpu_vi_poll_ack(adev);
> +	if (r)
> +		return r;
> +
> +	xgpu_vi_mailbox_set_valid(adev, false);
> +
> +	/* start to check msg if request is idh_req_gpu_init_access */
> +	if (request == IDH_REQ_GPU_INIT_ACCESS) {
> +		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
> +		if (r)
> +			return r;
> +	}
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_request_reset(struct amdgpu_device *adev)
> +{
> +	return xgpu_vi_send_access_requests(adev,
> IDH_REQ_GPU_RESET_ACCESS);
> +}
> +
> +static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +
> +	event = init ? IDH_REQ_GPU_INIT_ACCESS :
> IDH_REQ_GPU_FINI_ACCESS;
> +	return xgpu_vi_send_access_requests(adev, event);
> +}
> +
> +static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
> +					   bool init)
> +{
> +	enum idh_event event;
> +	int r = 0;
> +
> +	event = init ? IDH_REL_GPU_INIT_ACCESS :
> IDH_REL_GPU_FINI_ACCESS;
> +	r = xgpu_vi_send_access_requests(adev, event);
> +
> +	return r;
> +}
> +
> +/* add support mailbox interrupts */
> +static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	DRM_DEBUG("get ack intr and do nothing.\n");
> +	return 0;
> +}
> +
> +static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state)
> +{
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1
> : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
> +{
> +	struct amdgpu_virt *virt = container_of(work,
> +					struct amdgpu_virt, flr_work.work);
> +	struct amdgpu_device *adev = container_of(virt,
> +					struct amdgpu_device, virt);
> +	int r = 0;
> +
> +	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
> +	if (r)
> +		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
> +
> +	/* TODO: need to restore gfx states */
> +}
> +
> +static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				       struct amdgpu_irq_src *src,
> +				       unsigned type,
> +				       enum amdgpu_interrupt_state state)
> +{
> +	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
> +
> +	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
> +				 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1
> : 0);
> +	WREG32(mmMAILBOX_INT_CNTL, tmp);
> +
> +	return 0;
> +}
> +
> +static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
> +				   struct amdgpu_irq_src *source,
> +				   struct amdgpu_iv_entry *entry)
> +{
> +	int r;
> +
> +	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
> +	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
> +	/* do nothing for other msg */
> +	if (r)
> +		return 0;
> +
> +	/* TODO: need to save gfx states */
> +	schedule_delayed_work(&adev->virt.flr_work,
> +			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
> +
> +	return 0;
> +}
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_ack_irq,
> +	.process = xgpu_vi_mailbox_ack_irq,
> +};
> +
> +static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs =
> {
> +	.set = xgpu_vi_set_mailbox_rcv_irq,
> +	.process = xgpu_vi_mailbox_rcv_irq,
> +};
> +
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev)
> +{
> +	adev->virt.ack_irq.num_types = 1;
> +	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
> +	adev->virt.rcv_irq.num_types = 1;
> +	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs;
> +}
> +
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev)
> +{
> +	int r;
> +
> +	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
> +	if (r)
> +		return r;
> +
> +	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	return 0;
> +}
> +
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev)
> +{
> +	int r;
> +
> +	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
> +	if (r)
> +		return r;
> +	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
> +	if (r) {
> +		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +		return r;
> +	}
> +
> +	INIT_DELAYED_WORK(&adev->virt.flr_work,
> xgpu_vi_mailbox_flr_work);
> +
> +	return 0;
> +}
> +
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev)
> +{
> +	cancel_delayed_work_sync(&adev->virt.flr_work);
> +	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
> +	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
> +}
> +
> +const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
> +	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
> +	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
> +	.reset_gpu		= xgpu_vi_request_reset,
> +};
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> new file mode 100644
> index 0000000..fd6216e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
> @@ -0,0 +1,55 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
> EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
> THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef __MXGPU_VI_H__
> +#define __MXGPU_VI_H__
> +
> +#define VI_MAILBOX_TIMEDOUT	150
> +#define VI_MAILBOX_RESET_TIME	12
> +
> +/* VI mailbox messages request */
> +enum idh_request {
> +	IDH_REQ_GPU_INIT_ACCESS	= 1,
> +	IDH_REL_GPU_INIT_ACCESS,
> +	IDH_REQ_GPU_FINI_ACCESS,
> +	IDH_REL_GPU_FINI_ACCESS,
> +	IDH_REQ_GPU_RESET_ACCESS
> +};
> +
> +/* VI mailbox messages data */
> +enum idh_event {
> +	IDH_CLR_MSG_BUF = 0,
> +	IDH_READY_TO_ACCESS_GPU,
> +	IDH_FLR_NOTIFICATION,
> +	IDH_FLR_NOTIFICATION_CMPL,
> +	IDH_EVENT_MAX
> +};
> +
> +extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
> +
> +void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
> +void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev);
> +int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev);
> +int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev);
> +void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
> +
> +#endif
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 07/11] drm/amdgpu: request/release full gpu access if device is vf

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 07/11] drm/amdgpu: request/release full gpu access if device is
> vf
> 
> For gpu vf device, first need to request full gpu access before
> accessing gpu registers, and release full gpu access after the
> access is done.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 9 +++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 6 ++++++
>  2 files changed, 15 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 9a2fd3e..03d28a1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -1330,6 +1330,12 @@ static int amdgpu_early_init(struct
> amdgpu_device *adev)
>  		return -EINVAL;
>  	}
> 
> +	if (amdgpu_sriov_vf(adev)) {
> +		r = amdgpu_virt_request_full_gpu(adev, true);
> +		if (r)
> +			return r;
> +	}
> +
>  	for (i = 0; i < adev->num_ip_blocks; i++) {
>  		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
>  			DRM_ERROR("disabled ip block: %d\n", i);
> @@ -1523,6 +1529,9 @@ static int amdgpu_fini(struct amdgpu_device
> *adev)
>  		adev->ip_blocks[i].status.late_initialized = false;
>  	}
> 
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_release_full_gpu(adev, false);
> +
>  	return 0;
>  }
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 47bc8e1..eb228c7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -60,6 +60,9 @@ int amdgpu_driver_unload_kms(struct drm_device
> *dev)
>  	if (adev->rmmio == NULL)
>  		goto done_free;
> 
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_request_full_gpu(adev, false);
> +
>  	if (amdgpu_device_is_px(dev)) {
>  		pm_runtime_get_sync(dev->dev);
>  		pm_runtime_forbid(dev->dev);
> @@ -139,6 +142,9 @@ int amdgpu_driver_load_kms(struct drm_device
> *dev, unsigned long flags)
>  		pm_runtime_put_autosuspend(dev->dev);
>  	}
> 
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_release_full_gpu(adev, true);
> +
>  out:
>  	if (r) {
>  		/* balance pm_runtime_get_sync in
> amdgpu_driver_unload_kms */
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 08/11] drm/amdgpu/vi: add support virtualization

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 08/11] drm/amdgpu/vi: add support virtualization
> 
> Call VI virtualization functions if device is Vf.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/vi.c | 34
> ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> b/drivers/gpu/drm/amd/amdgpu/vi.c
> index dc0d4fa..34aa603 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -77,6 +77,7 @@
>  #include "amdgpu_dm.h"
>  #include "amdgpu_powerplay.h"
>  #include "dce_virtual.h"
> +#include "mxgpu_vi.h"
> 
>  /*
>   * Indirect registers accessor
> @@ -273,6 +274,12 @@ static void vi_init_golden_registers(struct
> amdgpu_device *adev)
>  	/* Some of the registers might be dependent on GRBM_GFX_INDEX
> */
>  	mutex_lock(&adev->grbm_idx_mutex);
> 
> +	if (amdgpu_sriov_vf(adev)) {
> +		xgpu_vi_init_golden_registers(adev);
> +		mutex_unlock(&adev->grbm_idx_mutex);
> +		return;
> +	}
> +
>  	switch (adev->asic_type) {
>  	case CHIP_TOPAZ:
>  		amdgpu_program_register_sequence(adev,
> @@ -892,8 +899,10 @@ static int vi_common_early_init(void *handle)
>  		(amdgpu_ip_block_mask & (1 <<
> AMD_IP_BLOCK_TYPE_SMC)))
>  		smc_enabled = true;
> 
> -	if (amdgpu_sriov_vf(adev))
> +	if (amdgpu_sriov_vf(adev)) {
>  		amdgpu_virt_init_setting(adev);
> +		xgpu_vi_mailbox_set_irq_funcs(adev);
> +	}
> 
>  	adev->rev_id = vi_get_rev_id(adev);
>  	adev->external_rev_id = 0xFF;
> @@ -1059,8 +1068,23 @@ static int vi_common_early_init(void *handle)
>  	return 0;
>  }
> 
> +static int vi_common_late_init(void *handle)
> +{
> +	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +
> +	if (amdgpu_sriov_vf(adev))
> +		xgpu_vi_mailbox_get_irq(adev);
> +
> +	return 0;
> +}
> +
>  static int vi_common_sw_init(void *handle)
>  {
> +	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +
> +	if (amdgpu_sriov_vf(adev))
> +		xgpu_vi_mailbox_add_irq_id(adev);
> +
>  	return 0;
>  }
> 
> @@ -1092,6 +1116,9 @@ static int vi_common_hw_fini(void *handle)
>  	/* enable the doorbell aperture */
>  	vi_enable_doorbell_aperture(adev, false);
> 
> +	if (amdgpu_sriov_vf(adev))
> +		xgpu_vi_mailbox_put_irq(adev);
> +
>  	return 0;
>  }
> 
> @@ -1376,7 +1403,7 @@ static int vi_common_set_powergating_state(void
> *handle,
>  static const struct amd_ip_funcs vi_common_ip_funcs = {
>  	.name = "vi_common",
>  	.early_init = vi_common_early_init,
> -	.late_init = NULL,
> +	.late_init = vi_common_late_init,
>  	.sw_init = vi_common_sw_init,
>  	.sw_fini = vi_common_sw_fini,
>  	.hw_init = vi_common_hw_init,
> @@ -1404,6 +1431,9 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
>  	/* in early init stage, vbios code won't work */
>  	vi_detect_hw_virtualization(adev);
> 
> +	if (amdgpu_sriov_vf(adev))
> +		adev->virt.ops = &xgpu_vi_virt_ops;
> +
>  	switch (adev->asic_type) {
>  	case CHIP_TOPAZ:
>  		/* topaz has no DCE, UVD, VCE */
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 09/11] drm/amdgpu/virt: enable virtual display

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 09/11] drm/amdgpu/virt: enable virtual display
> 
> Virtual display is default setting for virtualization, enable it.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index f32a789..5baca9f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -25,6 +25,9 @@
> 
>  void amdgpu_virt_init_setting(struct amdgpu_device *adev)
>  {
> +	/* enable virtual display */
> +	adev->mode_info.num_crtc = 1;
> +

Should probably also set adev->enable_virtual_display = true here.  With that fixed:
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

As a follow on patch, it would be nice to rework amdgpu_device_enable_virtual_display() to set some sensible defaults in the sr-iov case, so it's all handled in one place.

Alex

>  	mutex_init(&adev->virt.lock);
>  }
> 
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 10/11] drm/amdgpu/virt: disable dc

[Deucher, Alexander]

> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> Of Xiangliang Yu
> Sent: Tuesday, January 10, 2017 5:01 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang
> Subject: [V2 10/11] drm/amdgpu/virt: disable dc
> 
> Virtualization don't need the dc, disable it.
> 
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 03d28a1..a6d828c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum
> amd_asic_type asic_type)
>   */
>  bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
>  {
> +	if (amdgpu_sriov_vf(adev))
> +		return false;
> +
>  	return amdgpu_device_asic_has_dc_support(adev->asic_type);
>  }
> 
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg function

[Yu, Xiangliang]

> -----Original Message-----
> From: Deucher, Alexander
> Sent: Tuesday, January 10, 2017 11:35 PM
> To: Yu, Xiangliang <Xiangliang.Yu@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Yu, Xiangliang <Xiangliang.Yu@amd.com>
> Subject: RE: [V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg
> function
> 
> > -----Original Message-----
> > From: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] On Behalf
> > Of Xiangliang Yu	
> > Sent: Tuesday, January 10, 2017 5:01 AM
> > To: amd-gfx@lists.freedesktop.org
> > Cc: Yu, Xiangliang
> > Subject: [V2 02/11] drm/amdgpu/gfx8: implement emit_rreg/wreg
> function
> >
> > Implement emit_rreg/wreg function for kiq ring.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  3 ++-
> >  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 37
> > ++++++++++++++++++++++++++++++++
> >  2 files changed, 39 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 0d821d9..a6524bb 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -30,7 +30,8 @@
> >  #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU
> is
> > pass through for VM */
> >  /* GPU virtualization */
> >  struct amdgpu_virt {
> > -	uint32_t caps;
> > +	uint32_t		caps;
> > +	uint32_t		val_offs;
> 
> Maybe rename val_offs to register_val_offs to make it clearer what it is used
> for.  With that change, the patch is:
> Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

Ok. I'll fix it and send out V3 tomorrow.

> 
> >  };
> >
> >  #define amdgpu_sriov_enabled(adev) \
> > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> > index b487b97..e619f49 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> > @@ -1373,6 +1373,12 @@ static int gfx_v8_0_kiq_init_ring(struct
> > amdgpu_device *adev,  {
> >  	int r = 0;
> >
> > +	if (amdgpu_sriov_vf(adev)) {
> > +		r = amdgpu_wb_get(adev, &adev->virt.val_offs);
> > +		if (r)
> > +			return r;
> > +	}
> > +
> >  	ring->adev = NULL;
> >  	ring->ring_obj = NULL;
> >  	ring->use_doorbell = true;
> > @@ -1399,6 +1405,9 @@ static int gfx_v8_0_kiq_init_ring(struct
> > amdgpu_device *adev,  static void gfx_v8_0_kiq_free_ring(struct
> > amdgpu_ring *ring,
> >  				   struct amdgpu_irq_src *irq)
> >  {
> > +	if (amdgpu_sriov_vf(ring->adev))
> > +		amdgpu_wb_free(ring->adev, ring->adev->virt.val_offs);
> > +
> >  	amdgpu_ring_fini(ring);
> >  	irq->data = NULL;
> >  }
> > @@ -6681,6 +6690,32 @@ static void gfx_v8_ring_emit_cntxcntl(struct
> > amdgpu_ring *ring, uint32_t flags)
> >  	amdgpu_ring_write(ring, 0);
> >  }
> >
> > +static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring,
> > +uint32_t
> > reg)
> > +{
> > +	struct amdgpu_device *adev = ring->adev;
> > +
> > +	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
> > +	amdgpu_ring_write(ring, 0 |	/* src: register*/
> > +				(5 << 8) |	/* dst: memory */
> > +				(1 << 20));	/* write confirm */
> > +	amdgpu_ring_write(ring, reg);
> > +	amdgpu_ring_write(ring, 0);
> > +	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
> > +				adev->virt.val_offs * 4));
> > +	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
> > +				adev->virt.val_offs * 4));
> > +}
> > +
> > +static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring,
> > +uint32_t
> > reg,
> > +				  uint32_t val)
> > +{
> > +	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
> > +	amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
> > +	amdgpu_ring_write(ring, reg);
> > +	amdgpu_ring_write(ring, 0);
> > +	amdgpu_ring_write(ring, val);
> > +}
> > +
> >  static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device
> > *adev,
> >  						 enum
> > amdgpu_interrupt_state state)
> >  {
> > @@ -6995,6 +7030,8 @@ static const struct amdgpu_ring_funcs
> > gfx_v8_0_ring_funcs_kiq = {
> >  	.test_ib = gfx_v8_0_ring_test_ib,
> >  	.insert_nop = amdgpu_ring_insert_nop,
> >  	.pad_ib = amdgpu_ring_generic_pad_ib,
> > +	.emit_rreg = gfx_v8_0_ring_emit_rreg,
> > +	.emit_wreg = gfx_v8_0_ring_emit_wreg,
> >  };
> >
> >  static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
> > --
> > 2.7.4
> >
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

[ML] without those grab and release pare, it will trigger kernel page fault (in fence_wait) if the fence is freed before 
It signaled.


-----邮件原件-----
发件人: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] 代表 Christian König
发送时间: Tuesday, January 10, 2017 9:09 PM
收件人: Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access 
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 101 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>   	uint32_t ret;
>   
> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +		return amdgpu_virt_kiq_rreg(adev, reg);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>   
> +	if (amdgpu_sriov_runtime(adev))
> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..6520a4e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,86 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person 
> +obtaining a
> + * copy of this software and associated documentation files (the 
> +"Software"),
> + * to deal in the Software without restriction, including without 
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute, 
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom 
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be 
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> +	mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t 
> +reg) {
> +	signed long r;
> +	uint32_t val;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_rreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);	
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +
> +	val = adev->wb.wb[adev->virt.val_offs];
> +
> +	return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v) {
> +	signed long r;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_wreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_wreg(ring, reg, v);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 79619b6..24f0590 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		val_offs;
> +	struct mutex		lock;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline 
> bool is_virtual_machine(void)
>   #endif
>   }
>   
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c 
> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>   		smc_enabled = true;
>   
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_setting(adev);
> +
>   	adev->rev_id = vi_get_rev_id(adev);
>   	adev->external_rev_id = 0xFF;
>   	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

[ML] e.g.  without those grab/release pare, if fence is signaled right after ring_commit(), then
"amdgpu_fence_process" will be invoked by interrupt and it will call fence_put() on this fence, so this 
Fence is never valid and all fence_wait on it in sequence will trigger page fault ...

BR Monk



-----邮件原件-----
发件人: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] 代表 Christian König
发送时间: Tuesday, January 10, 2017 9:09 PM
收件人: Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access 
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 101 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile 
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>   
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>   	uint32_t ret;
>   
> +	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +		return amdgpu_virt_kiq_rreg(adev, reg);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>   
> +	if (amdgpu_sriov_runtime(adev))
> +		return amdgpu_virt_kiq_wreg(adev, reg, v);
> +
>   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
>   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>   	else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..6520a4e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,86 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person 
> +obtaining a
> + * copy of this software and associated documentation files (the 
> +"Software"),
> + * to deal in the Software without restriction, including without 
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute, 
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom 
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be 
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> +	mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t 
> +reg) {
> +	signed long r;
> +	uint32_t val;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_rreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_rreg(ring, reg);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +
> +	val = adev->wb.wb[adev->virt.val_offs];
> +
> +	return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v) {
> +	signed long r;
> +	struct fence *f;
> +	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +	struct amdgpu_ring *ring = &kiq->ring;
> +
> +	BUG_ON(!ring->funcs->emit_wreg);
> +
> +	mutex_lock(&adev->virt.lock);
> +	amdgpu_ring_alloc(ring, 32);
> +	amdgpu_ring_emit_hdp_flush(ring);
> +	amdgpu_ring_emit_wreg(ring, reg, v);
> +	amdgpu_ring_emit_hdp_invalidate(ring);
> +	amdgpu_fence_emit(ring, &f);
> +	fence_get(f);
> +	amdgpu_ring_commit(ring);
> +	mutex_unlock(&adev->virt.lock);
> +
> +	r = fence_wait(f, false);
> +	fence_put(f);
> +	if (r)
> +		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +	fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 79619b6..24f0590 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>   	uint32_t		caps;
>   	uint32_t		val_offs;
> +	struct mutex		lock;
>   };
>   
>   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline 
> bool is_virtual_machine(void)
>   #endif
>   }
>   
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
> +uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c 
> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>   		smc_enabled = true;
>   
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_setting(adev);
> +
>   	adev->rev_id = vi_get_rev_id(adev);
>   	adev->external_rev_id = 0xFF;
>   	switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Liu, Monk]

[-- Attachment #1.1: Type: text/plain, Size: 9987 bytes --]

Hi Christian


latest updates on the fence grab/release


I recall the impression of why I added those grab/release pare:


I checked the history, the first time I added those pare is because they are first introduced/implemented in amd-sriov-4.3, and that branch's fence implement has big gap between current (4.3 fence doesn't use RCU and kmem_cache)


in amd-sriov-4.3 branch, without this grab/release around I found there will be kernel page fault triggered if fence quickly signaled before fence_wait(), because amdgpu_fence_process will put the fence and lead to fence released prior to later's fence_wait().


but for staging-4.7 code, I didn't found such page fault after remove those grab/release, and I noticed that the reason maybe we increases the kref of fence to 2 in amdgpu_fence_emit ( in 4.3 kernel we only increase it to 1):


Increasing to 1 is by fence_init, and to 2 is by rcu_assign_pointer(*ptr,fence_get(&fence->base))

and with this reason the fence is 1 after it signaled. Agree  with you that we should remove those grab/release and only leave one fence_put() in kiq reg access routines.


I didn't check the code on detail previously, thanks !


BR Monk


________________________________
发件人: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> 代表 Liu, Monk <Monk.Liu@amd.com>
发送时间: 2017年1月11日 11:06:34
收件人: Christian König; Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: 答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

> +     fence_get(f);
> +     amdgpu_ring_commit(ring);
> +     mutex_unlock(&adev->virt.lock);
> +
> +     r = fence_wait(f, false);
> +     fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

[ML] e.g.  without those grab/release pare, if fence is signaled right after ring_commit(), then
"amdgpu_fence_process" will be invoked by interrupt and it will call fence_put() on this fence, so this
Fence is never valid and all fence_wait on it in sequence will trigger page fault ...

BR Monk



-----邮件原件-----
发件人: amd-gfx [mailto:amd-gfx-bounces@lists.freedesktop.org] 代表 Christian König
发送时间: Tuesday, January 10, 2017 9:09 PM
收件人: Yu, Xiangliang; amd-gfx@lists.freedesktop.org
主题: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> For virtualization, it is must for driver to use KIQ to access
> registers when it is out of GPU full access mode.
>
> Signed-off-by: Xiangliang Yu <Xiangliang.Yu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 ++++++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
>   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
>   5 files changed, 101 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> b/drivers/gpu/drm/amd/amdgpu/Makefile
> index 4185b03..0b8e470 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>        atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
>        amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
>        amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> -     amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> +     amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
>
>   # add asic specific block
>   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f82919d..9a2fd3e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
>   {
>        uint32_t ret;
>
> +     if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> +             return amdgpu_virt_kiq_rreg(adev, reg);
> +
>        if ((reg * 4) < adev->rmmio_size && !always_indirect)
>                ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
>        else {
> @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
>   {
>        trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
>
> +     if (amdgpu_sriov_runtime(adev))
> +             return amdgpu_virt_kiq_wreg(adev, reg, v);
> +
>        if ((reg * 4) < adev->rmmio_size && !always_indirect)
>                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
>        else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> new file mode 100644
> index 0000000..6520a4e
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -0,0 +1,86 @@
> +/*
> + * Copyright 2017 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person
> +obtaining a
> + * copy of this software and associated documentation files (the
> +"Software"),
> + * to deal in the Software without restriction, including without
> +limitation
> + * the rights to use, copy, modify, merge, publish, distribute,
> +sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom
> +the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be
> +included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> +EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> +MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT
> +SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> +DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> +OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
> +OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "amdgpu.h"
> +#include "amdgpu_virt.h"
> +
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> +     mutex_init(&adev->virt.lock);
> +}
> +
> +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> +reg) {
> +     signed long r;
> +     uint32_t val;
> +     struct fence *f;
> +     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +     struct amdgpu_ring *ring = &kiq->ring;
> +
> +     BUG_ON(!ring->funcs->emit_rreg);
> +
> +     mutex_lock(&adev->virt.lock);
> +     amdgpu_ring_alloc(ring, 32);
> +     amdgpu_ring_emit_hdp_flush(ring);
> +     amdgpu_ring_emit_rreg(ring, reg);
> +     amdgpu_ring_emit_hdp_invalidate(ring);
> +     amdgpu_fence_emit(ring, &f);
> +     fence_get(f);
> +     amdgpu_ring_commit(ring);
> +     mutex_unlock(&adev->virt.lock);
> +
> +     r = fence_wait(f, false);
> +     fence_put(f);

Why do you grab and release an extra fence reference here?

Christian.

> +     if (r)
> +             DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +     fence_put(f);
> +
> +     val = adev->wb.wb[adev->virt.val_offs];
> +
> +     return val;
> +}
> +
> +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> +uint32_t v) {
> +     signed long r;
> +     struct fence *f;
> +     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> +     struct amdgpu_ring *ring = &kiq->ring;
> +
> +     BUG_ON(!ring->funcs->emit_wreg);
> +
> +     mutex_lock(&adev->virt.lock);
> +     amdgpu_ring_alloc(ring, 32);
> +     amdgpu_ring_emit_hdp_flush(ring);
> +     amdgpu_ring_emit_wreg(ring, reg, v);
> +     amdgpu_ring_emit_hdp_invalidate(ring);
> +     amdgpu_fence_emit(ring, &f);
> +     fence_get(f);
> +     amdgpu_ring_commit(ring);
> +     mutex_unlock(&adev->virt.lock);
> +
> +     r = fence_wait(f, false);
> +     fence_put(f);
> +     if (r)
> +             DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> +     fence_put(f);
> +}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index 79619b6..24f0590 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -33,6 +33,7 @@
>   struct amdgpu_virt {
>        uint32_t                caps;
>        uint32_t                val_offs;
> +     struct mutex            lock;
>   };
>
>   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline
> bool is_virtual_machine(void)
>   #endif
>   }
>
> +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
> +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
> +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> +uint32_t v);
> +
>   #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
>                (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
>                smc_enabled = true;
>
> +     if (amdgpu_sriov_vf(adev))
> +             amdgpu_virt_init_setting(adev);
> +
>        adev->rev_id = vi_get_rev_id(adev);
>        adev->external_rev_id = 0xFF;
>        switch (adev->asic_type) {


_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[-- Attachment #1.2: Type: text/html, Size: 16558 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: 答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers

[Christian König]

[-- Attachment #1.1: Type: text/plain, Size: 11059 bytes --]

Ah, that issue again. Yeah that was fixed in the meantime. Please remove 
the extra fance_get()/fence_put() for upstreaming.

If an old branch still has that issue we should backport the proper 
fixes and not work around them like this.

Thanks for looking into it,
Christian.

Am 11.01.2017 um 09:30 schrieb Liu, Monk:
>
> Hi Christian
>
>
> latest updates on the fence grab/release
>
>
> I recall the impression of why I added those grab/release pare:
>
>
> I checked the history, the first time I added those pare is because 
> they are first introduced/implemented in amd-sriov-4.3, and that 
> branch's fence implement has big gap between current (4.3 fence 
> doesn't use RCU and kmem_cache)
>
>
> in amd-sriov-4.3 branch, without this grab/release around I found 
> there will be kernel page fault triggered if fence quickly signaled 
> before fence_wait(), because amdgpu_fence_process will put the fence 
> and lead to fence released prior to later's fence_wait().
>
>
> but for staging-4.7 code, I didn't found such page fault after remove 
> those grab/release, and I noticed that the reason maybe we increases 
> the kref of fence to 2 in amdgpu_fence_emit ( in 4.3 kernel we only 
> increase it to 1):
>
>
> Increasing to 1 is by fence_init, and to 2 is by 
> rcu_assign_pointer(*ptr,fence_get(&fence->base))
>
>
> and with this reason the fence is 1 after it signaled. Agree  with you 
> that we should remove those grab/release and only leave one 
> fence_put() in kiq reg access routines.
>
>
> I didn't check the code on detail previously, thanks !
>
>
> BR Monk
>
>
> ------------------------------------------------------------------------
> *发件人:* amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> 代表 Liu, Monk 
> <Monk.Liu-5C7GfCeVMHo@public.gmane.org>
> *发送时间:* 2017年1月11日 11:06:34
> *收件人:* Christian König; Yu, Xiangliang; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> *主题:* 答复: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
> > +     fence_get(f);
> > +     amdgpu_ring_commit(ring);
> > +     mutex_unlock(&adev->virt.lock);
> > +
> > +     r = fence_wait(f, false);
> > +     fence_put(f);
>
> Why do you grab and release an extra fence reference here?
>
> Christian.
>
> [ML] e.g.  without those grab/release pare, if fence is signaled right 
> after ring_commit(), then
> "amdgpu_fence_process" will be invoked by interrupt and it will call 
> fence_put() on this fence, so this
> Fence is never valid and all fence_wait on it in sequence will trigger 
> page fault ...
>
> BR Monk
>
>
>
> -----邮件原件-----
> 发件人: amd-gfx [mailto:amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org] 代表 
> Christian König
> 发送时间: Tuesday, January 10, 2017 9:09 PM
> 收件人: Yu, Xiangliang; amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> 主题: Re: [V2 04/11] drm/amdgpu/virt: use kiq to access registers
>
> Am 10.01.2017 um 11:00 schrieb Xiangliang Yu:
> > For virtualization, it is must for driver to use KIQ to access
> > registers when it is out of GPU full access mode.
> >
> > Signed-off-by: Xiangliang Yu <Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 86 
> ++++++++++++++++++++++++++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
> >   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
> >   5 files changed, 101 insertions(+), 1 deletion(-)
> >   create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile
> > b/drivers/gpu/drm/amd/amdgpu/Makefile
> > index 4185b03..0b8e470 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/Makefile
> > +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
> > @@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
> >        atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
> >        amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
> >        amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
> > -     amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
> > +     amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
> >
> >   # add asic specific block
> >   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > index f82919d..9a2fd3e 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> > @@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device 
> *adev, uint32_t reg,
> >   {
> >        uint32_t ret;
> >
> > +     if (amdgpu_sriov_runtime(adev) && !in_interrupt())
> > +             return amdgpu_virt_kiq_rreg(adev, reg);
> > +
> >        if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >                ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
> >        else {
> > @@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, 
> uint32_t reg, uint32_t v,
> >   {
> >        trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
> >
> > +     if (amdgpu_sriov_runtime(adev))
> > +             return amdgpu_virt_kiq_wreg(adev, reg, v);
> > +
> >        if ((reg * 4) < adev->rmmio_size && !always_indirect)
> >                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
> >        else {
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > new file mode 100644
> > index 0000000..6520a4e
> > --- /dev/null
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> > @@ -0,0 +1,86 @@
> > +/*
> > + * Copyright 2017 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be
> > +included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT
> > +SHALL
> > + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
> > +DAMAGES OR
> > + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> > +OTHERWISE,
> > + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
> > +OR
> > + * OTHER DEALINGS IN THE SOFTWARE.
> > + */
> > +
> > +#include "amdgpu.h"
> > +#include "amdgpu_virt.h"
> > +
> > +void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
> > +     mutex_init(&adev->virt.lock);
> > +}
> > +
> > +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
> > +reg) {
> > +     signed long r;
> > +     uint32_t val;
> > +     struct fence *f;
> > +     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > +     struct amdgpu_ring *ring = &kiq->ring;
> > +
> > +     BUG_ON(!ring->funcs->emit_rreg);
> > +
> > +     mutex_lock(&adev->virt.lock);
> > +     amdgpu_ring_alloc(ring, 32);
> > +     amdgpu_ring_emit_hdp_flush(ring);
> > +     amdgpu_ring_emit_rreg(ring, reg);
> > +     amdgpu_ring_emit_hdp_invalidate(ring);
> > +     amdgpu_fence_emit(ring, &f);
> > +     fence_get(f);
> > +     amdgpu_ring_commit(ring);
> > +     mutex_unlock(&adev->virt.lock);
> > +
> > +     r = fence_wait(f, false);
> > +     fence_put(f);
>
> Why do you grab and release an extra fence reference here?
>
> Christian.
>
> > +     if (r)
> > +             DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> > +     fence_put(f);
> > +
> > +     val = adev->wb.wb[adev->virt.val_offs];
> > +
> > +     return val;
> > +}
> > +
> > +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > +uint32_t v) {
> > +     signed long r;
> > +     struct fence *f;
> > +     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
> > +     struct amdgpu_ring *ring = &kiq->ring;
> > +
> > +     BUG_ON(!ring->funcs->emit_wreg);
> > +
> > +     mutex_lock(&adev->virt.lock);
> > +     amdgpu_ring_alloc(ring, 32);
> > +     amdgpu_ring_emit_hdp_flush(ring);
> > +     amdgpu_ring_emit_wreg(ring, reg, v);
> > +     amdgpu_ring_emit_hdp_invalidate(ring);
> > +     amdgpu_fence_emit(ring, &f);
> > +     fence_get(f);
> > +     amdgpu_ring_commit(ring);
> > +     mutex_unlock(&adev->virt.lock);
> > +
> > +     r = fence_wait(f, false);
> > +     fence_put(f);
> > +     if (r)
> > +             DRM_ERROR("wait for kiq fence error: %ld.\n", r);
> > +     fence_put(f);
> > +}
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > index 79619b6..24f0590 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> > @@ -33,6 +33,7 @@
> >   struct amdgpu_virt {
> >        uint32_t                caps;
> >        uint32_t                val_offs;
> > +     struct mutex            lock;
> >   };
> >
> >   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline
> > bool is_virtual_machine(void)
> >   #endif
> >   }
> >
> > +void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
> > +amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
> > +amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
> > +uint32_t v);
> > +
> >   #endif
> > diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c
> > b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/vi.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/vi.c
> > @@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
> >                (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
> >                smc_enabled = true;
> >
> > +     if (amdgpu_sriov_vf(adev))
> > +             amdgpu_virt_init_setting(adev);
> > +
> >        adev->rev_id = vi_get_rev_id(adev);
> >        adev->external_rev_id = 0xFF;
> >        switch (adev->asic_type) {
>
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx



[-- Attachment #1.2: Type: text/html, Size: 19761 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx