* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-08 10:32 Tong Liu01
2022-11-08 10:40 ` Liu01, Tong (Esther)
0 siblings, 1 reply; 27+ messages in thread
From: Tong Liu01 @ 2022-11-08 10:32 UTC (permalink / raw)
To: amd-gfx
Cc: Jack Xiao, Feifei Xu, horace.chen, Kevin Wang, Tong Liu01,
Tuikov Luben, Deucher Alexander, Evan Quan, Christian König,
Monk Liu, Hawking Zhang
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR
region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
4 files changed, 192 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..032dc2678d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, fw_size, drv_size;
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+ drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
+ fw_size,
+ drv_size);
+
+ if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes = drv_size << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
+ fw_size,
+ drv_start_addr,
+ drv_size);
+
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..4a73cb314086 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..c0f56ae653f0 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 10:32 [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2 Tong Liu01
@ 2022-11-08 10:40 ` Liu01, Tong (Esther)
2022-11-08 12:53 ` Christian König
0 siblings, 1 reply; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-08 10:40 UTC (permalink / raw)
To: Liu01, Tong (Esther), amd-gfx, Koenig, Christian
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Chang, HaiJun, Tuikov, Luben, Sohail,
Rashid, Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang,
Hawking
[AMD Official Use Only - General]
Hi @Koenig, Christian,
Refined as your comment. By the way:
if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT))
This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
Kind regards,
Esther
-----Original Message-----
From: Tong Liu01 <Tong.Liu01@amd.com>
Sent: 2022年11月8日星期二 下午6:33
To: amd-gfx@lists.freedesktop.org
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
4 files changed, 192 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..032dc2678d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, fw_size, drv_size;
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+ drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
+ fw_size,
+ drv_size);
+
+ if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes = drv_size << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size =
+le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
+ fw_size,
+ drv_start_addr,
+ drv_size);
+
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..4a73cb314086 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
+*adev) {
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
+driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..c0f56ae653f0 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 10:40 ` Liu01, Tong (Esther)
@ 2022-11-08 12:53 ` Christian König
2022-11-08 13:49 ` Chang, HaiJun
0 siblings, 1 reply; 27+ messages in thread
From: Christian König @ 2022-11-08 12:53 UTC (permalink / raw)
To: Liu01, Tong (Esther), amd-gfx
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Chang, HaiJun, Tuikov, Luben, Sohail,
Rashid, Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang,
Hawking
Hi Esther
well there are a couple of things which you need to address before
getting this merged.
First of all the patch you send out uses dos line endings instead of the
unix line endings. Not sure how you manage to do that, but please use
"git send-email" instead to avoid that.
Then your patch contains a bunch of white spaces after code warning
which checkpatch.pl complains about (after ignoring the dos line ending
warnings). So this was clearly not properly checked with checkpatch.pl.
Then the kernel coding style usually says that with a multi line "if ("
the next lines should start after the opening "(". In other words intend
with tabs and the whitespaces. I'm not sure what editor you are using,
but there are standard settings available for basically all large
editors which does stuff like that automatically. Please try to use one
of those.
Regarding the casing of the values it's a good argument that you only
move the code around, but the general coding style is just extremely
questionable. The defines should use the lowest necessary integer type.
But it's correct that this should probably be part of another patch.
Regards,
Christian.
Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi @Koenig, Christian,
>
> Refined as your comment. By the way:
> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>
> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Tong Liu01 <Tong.Liu01@amd.com>
> Sent: 2022年11月8日星期二 下午6:33
> To: amd-gfx@lists.freedesktop.org
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
> 4 files changed, 192 insertions(+), 31 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..032dc2678d7c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
> }
> }
>
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
> + int *usage_bytes)
> +{
> + uint32_t start_addr, fw_size, drv_size;
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
> + drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
> +
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + start_addr,
> + fw_size,
> + drv_size);
> +
> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + /* Use the default scratch size */
> + *usage_bytes = 0;
> + } else {
> + *usage_bytes = drv_size << 10;
> + }
> + return 0;
> +}
> +
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
> + int *usage_bytes)
> +{
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
> +
> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
> + drv_size =
> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
> +
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + fw_start_addr,
> + fw_size,
> + drv_start_addr,
> + drv_size);
> +
> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> + *usage_bytes = 0;
> + return 0;
> +}
> +
> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> - usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
> + &usage_bytes);
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
> + &usage_bytes);
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..4a73cb314086 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
> +*adev) {
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
> +driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> +{
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + NULL);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r)
> + return r;
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..339838675b11 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,11 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..c0f56ae653f0 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> - ***************************************************************************
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> {
> - struct atom_common_table_header table_header;
> - uint32_t start_address_in_kb;
> - uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + struct atom_common_table_header table_header;
> + uint32_t start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2 {
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
> ***************************************************************************
> --
> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 12:53 ` Christian König
@ 2022-11-08 13:49 ` Chang, HaiJun
2022-11-08 14:04 ` Christian König
0 siblings, 1 reply; 27+ messages in thread
From: Chang, HaiJun @ 2022-11-08 13:49 UTC (permalink / raw)
To: Koenig, Christian, Liu01, Tong (Esther), amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid,
Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang, Hawking
[AMD Official Use Only - General]
+ Bokun to help addressing the coding style problem in MKM side.
-----Original Message-----
From: Koenig, Christian <Christian.Koenig@amd.com>
Sent: Tuesday, November 8, 2022 8:53 PM
To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Hi Esther
well there are a couple of things which you need to address before getting this merged.
First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
Then the kernel coding style usually says that with a multi line "if ("
the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
But it's correct that this should probably be part of another patch.
Regards,
Christian.
Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi @Koenig, Christian,
>
> Refined as your comment. By the way:
> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>
> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Tong Liu01 <Tong.Liu01@amd.com>
> Sent: 2022年11月8日星期二 下午6:33
> To: amd-gfx@lists.freedesktop.org
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
> vram_usagebyfirmware_v2_2
>
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
> TMR region firstly to make sure TMR can be allocated at 2MB
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
> 4 files changed, 192 insertions(+), 31 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..032dc2678d7c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
> }
> }
>
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
> + int *usage_bytes)
> +{
> + uint32_t start_addr, fw_size, drv_size;
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
> + drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
> +
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + start_addr,
> + fw_size,
> + drv_size);
> +
> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + /* Use the default scratch size */
> + *usage_bytes = 0;
> + } else {
> + *usage_bytes = drv_size << 10;
> + }
> + return 0;
> +}
> +
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
> + int *usage_bytes)
> +{
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
> +
> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
> + drv_size =
> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
> +
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + fw_start_addr,
> + fw_size,
> + drv_start_addr,
> + drv_size);
> +
> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> + *usage_bytes = 0;
> + return 0;
> +}
> +
> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> - usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
> + &usage_bytes);
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
> + &usage_bytes);
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..4a73cb314086 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
> +*adev) {
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
> +driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
> +*adev) {
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + NULL);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r)
> + return r;
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..339838675b11 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,11 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
> b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..c0f56ae653f0 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> -
> **********************************************************************
> *****
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> {
> - struct atom_common_table_header table_header;
> - uint32_t start_address_in_kb;
> - uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + struct atom_common_table_header table_header;
> + uint32_t start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2 {
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
>
> **********************************************************************
> *****
> --
> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 13:49 ` Chang, HaiJun
@ 2022-11-08 14:04 ` Christian König
2022-11-08 16:33 ` Luben Tuikov
2022-11-10 10:14 ` Liu01, Tong (Esther)
0 siblings, 2 replies; 27+ messages in thread
From: Christian König @ 2022-11-08 14:04 UTC (permalink / raw)
To: Chang, HaiJun, Liu01, Tong (Esther), amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid, Deucher,
Alexander, Quan, Evan, Liu, Monk, Zhang, Hawking
Yeah, I mean the code looks correct.
It's just that style problems are usually pointed out by automated
checkers, especially things like dos line endings.
So get that fixed and we can push it immediately.
Thanks,
Christian.
Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
> [AMD Official Use Only - General]
>
> + Bokun to help addressing the coding style problem in MKM side.
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: Tuesday, November 8, 2022 8:53 PM
> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Hi Esther
>
> well there are a couple of things which you need to address before getting this merged.
>
> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>
> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>
> Then the kernel coding style usually says that with a multi line "if ("
> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>
> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
> But it's correct that this should probably be part of another patch.
>
> Regards,
> Christian.
>
> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>> [AMD Official Use Only - General]
>>
>> Hi @Koenig, Christian,
>>
>> Refined as your comment. By the way:
>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>
>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>
>> Kind regards,
>> Esther
>>
>> -----Original Message-----
>> From: Tong Liu01 <Tong.Liu01@amd.com>
>> Sent: 2022年11月8日星期二 下午6:33
>> To: amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
>> TMR region firstly to make sure TMR can be allocated at 2MB
>>
>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>> ---
>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> index b81b77a9efa6..032dc2678d7c 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>> }
>> }
>>
>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>> + int *usage_bytes)
>> +{
>> + uint32_t start_addr, fw_size, drv_size;
>> +
>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>> + drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>> +
>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>> + start_addr,
>> + fw_size,
>> + drv_size);
>> +
>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>> + /* Use the default scratch size */
>> + *usage_bytes = 0;
>> + } else {
>> + *usage_bytes = drv_size << 10;
>> + }
>> + return 0;
>> +}
>> +
>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>> + int *usage_bytes)
>> +{
>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>> +
>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>> +
>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>> + drv_size =
>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>> +
>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>> + fw_start_addr,
>> + fw_size,
>> + drv_start_addr,
>> + drv_size);
>> +
>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>> + }
>> +
>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>> + /* driver request VRAM reservation for SR-IOV */
>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>> + }
>> +
>> + *usage_bytes = 0;
>> + return 0;
>> +}
>> +
>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>> struct atom_context *ctx = adev->mode_info.atom_context;
>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>> vram_usagebyfirmware);
>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>> - uint32_t start_addr, size;
>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>> uint16_t data_offset;
>> + uint8_t frev, crev;
>> int usage_bytes = 0;
>>
>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>> -
>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>> -
>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> - /* Firmware request VRAM reservation for SR-IOV */
>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> - adev->mman.fw_vram_usage_size = size << 10;
>> - /* Use the default scratch size */
>> - usage_bytes = 0;
>> - } else {
>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>> + if (frev == 2 && crev == 1) {
>> + firmware_usage_v2_1 =
>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>> + firmware_usage_v2_1,
>> + &usage_bytes);
>> + } else if (frev >= 2 && crev >= 2) {
>> + firmware_usage_v2_2 =
>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>> + firmware_usage_v2_2,
>> + &usage_bytes);
>> }
>> }
>> +
>> ctx->scratch_size_bytes = 0;
>> if (usage_bytes == 0)
>> usage_bytes = 20 * 1024;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> index 585460ab8dfd..4a73cb314086 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>> NULL, &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/*
>> + * Driver Reservation functions
>> + */
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * free drv reserved vram if it has been reserved.
>> + */
>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>> +*adev) {
>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>> + NULL, NULL);
>> +}
>> +
>> /**
>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>> *
>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>> &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
>> +driver
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * create bo vram reservation from drv.
>> + */
>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>> +*adev) {
>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>> +
>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>> +
>> + if (adev->mman.drv_vram_usage_size == 0 ||
>> + adev->mman.drv_vram_usage_size > vram_size)
>> + return 0;
>> +
>> + return amdgpu_bo_create_kernel_at(adev,
>> + adev->mman.drv_vram_usage_start_offset,
>> + adev->mman.drv_vram_usage_size,
>> + AMDGPU_GEM_DOMAIN_VRAM,
>> + &adev->mman.drv_vram_usage_reserved_bo,
>> + NULL);
>> +}
>> +
>> /*
>> * Memoy training reservation functions
>> */
>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>> return r;
>> }
>>
>> + /*
>> + *The reserved vram for driver must be pinned to the specified
>> + *place on the VRAM, so reserve it early.
>> + */
>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>> + if (r)
>> + return r;
>> +
>> /*
>> * only NAVI10 and onwards ASIC support for IP discovery.
>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>> &adev->mman.sdma_access_ptr);
>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>
>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> index 9120ae80ef52..339838675b11 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>> void *fw_vram_usage_va;
>>
>> + /* driver VRAM reservation */
>> + u64 drv_vram_usage_start_offset;
>> + u64 drv_vram_usage_size;
>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>> +
>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>> struct amdgpu_bo *sdma_access_bo;
>> void *sdma_access_ptr;
>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>> index ff855cb21d3f..c0f56ae653f0 100644
>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>
>>
>> -/*
>> - ***************************************************************************
>> - Data Table vram_usagebyfirmware structure
>> -
>> **********************************************************************
>> *****
>> +/*
>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>> + driver can allocate driver reservation region under firmware reservation,
>> + used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>> + Host driver would overwrite the table with the following
>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>> + } else {
>> + there is no VBIOS reservation region
>> + driver must allocate driver reservation region at top of FB.
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + }
>> + } else { //( NV1X and after)
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>> + }
>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>> + driver can allocate driver reservation region under firmware reservation,
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + } else {
>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>> + driver set used_by_driver_region0_in_kb = driver reservation size
>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>> + }
>> + }
>> */
>>
>> struct vram_usagebyfirmware_v2_1
>> {
>> - struct atom_common_table_header table_header;
>> - uint32_t start_address_in_kb;
>> - uint16_t used_by_firmware_in_kb;
>> - uint16_t used_by_driver_in_kb;
>> + struct atom_common_table_header table_header;
>> + uint32_t start_address_in_kb;
>> + uint16_t used_by_firmware_in_kb;
>> + uint16_t used_by_driver_in_kb;
>> };
>>
>> +struct vram_usagebyfirmware_v2_2 {
>> + struct atom_common_table_header table_header;
>> + uint32_t fw_region_start_address_in_kb;
>> + uint16_t used_by_firmware_in_kb;
>> + uint16_t reserved;
>> + uint32_t driver_region0_start_address_in_kb;
>> + uint32_t used_by_driver_region0_in_kb;
>> + uint32_t reserved32[7];
>> +};
>>
>> /*
>>
>> **********************************************************************
>> *****
>> --
>> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 14:04 ` Christian König
@ 2022-11-08 16:33 ` Luben Tuikov
2022-11-10 5:25 ` Liu01, Tong (Esther)
2022-11-10 10:14 ` Liu01, Tong (Esther)
1 sibling, 1 reply; 27+ messages in thread
From: Luben Tuikov @ 2022-11-08 16:33 UTC (permalink / raw)
To: Christian König, Chang, HaiJun, Liu01, Tong (Esther),
amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Sohail, Rashid, Deucher, Alexander,
Quan, Evan, Liu, Monk, Zhang, Hawking
Hi,
Applying this patch to amd-staging-drm-next, with checkpatch.pl enabled, generates the following output. Perhaps those issues should be address and fixed.
$git am ~/patches/tongliu01/\[PATCH]\ drm_amdgpu\:\ add\ vram\ reservation\ logic\ based\ on\ vram_usagebyfirmware_v2_2\ -\ Tong\ Liu01\ \<Tong.Liu01\@amd.com\>\ -\ 2022-11-08\ 0532.eml
Applying: drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
-:10: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#10: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:105:
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
-:13: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#13: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:108:
+ uint32_t start_addr, fw_size, drv_size;
-:20: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#20: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:115:
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
-:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#40: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:135:
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
-:43: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#43: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:138:
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
-:45: WARNING:LONG_LINE: line length of 88 exceeds 81 columns
#45: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:140:
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
-:48: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#48: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:143:
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
-:49: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
#49: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:144:
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
-:51: WARNING:LONG_LINE_STRING: line length of 86 exceeds 81 columns
#51: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:146:
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
-:52: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#52: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:147:
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
-:57: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
#57: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:152:
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
-:64: WARNING:LONG_LINE: line length of 83 exceeds 81 columns
#64: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:159:
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
-:85: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u8' over 'uint8_t'
#85: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:178:
+ uint8_t frev, crev;
-:109: WARNING:LONG_LINE: line length of 90 exceeds 81 columns
#109: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:181:
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
-:112: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#112: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:184:
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
-:114: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#114: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:186:
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
-:118: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#118: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:190:
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
-:120: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#120: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:192:
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
-:149: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#149: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1595:
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
-:168: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u64' over 'uint64_t'
#168: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1633:
+ uint64_t vram_size = adev->gmc.visible_vram_size;
-:239: WARNING:LONG_LINE_COMMENT: line length of 113 exceeds 81 columns
#239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
-:239: WARNING:BLOCK_COMMENT_STYLE: Block comments use * on subsequent lines
#239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
-:240: WARNING:LONG_LINE_COMMENT: line length of 112 exceeds 81 columns
#240: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
-:245: WARNING:LONG_LINE_COMMENT: line length of 132 exceeds 81 columns
#245: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
-:248: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
#248: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:718:
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
-:249: WARNING:LONG_LINE_COMMENT: line length of 102 exceeds 81 columns
#249: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
-:257: WARNING:LONG_LINE_COMMENT: line length of 87 exceeds 81 columns
#257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:727:
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
-:262: WARNING:LONG_LINE_COMMENT: line length of 99 exceeds 81 columns
#262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:732:
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
-:263: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 81 columns
#263: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
-:268: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
#268: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:738:
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
-:271: WARNING:TYPO_SPELLING: 'dirver' may be misspelled - perhaps 'driver'?
#271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
^^^^^^
-:271: WARNING:LONG_LINE_COMMENT: line length of 104 exceeds 81 columns
#271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
-:273: WARNING:LONG_LINE_COMMENT: line length of 94 exceeds 81 columns
#273: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:743:
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
-:274: WARNING:LONG_LINE_COMMENT: line length of 97 exceeds 81 columns
#274: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:744:
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
-:275: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
#275: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:745:
+ as the reservation for VF as it doesn’t exist. And Host driver should also
-:276: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
#276: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:746:
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
-:288: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:754:
+ uint32_t start_address_in_kb;
-:289: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#289: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:755:
+ uint16_t used_by_firmware_in_kb;
-:290: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#290: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:756:
+ uint16_t used_by_driver_in_kb;
-:295: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#295: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:761:
+ uint32_t fw_region_start_address_in_kb;
-:296: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#296: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:762:
+ uint16_t used_by_firmware_in_kb;
-:297: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#297: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:763:
+ uint16_t reserved;
-:298: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#298: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:764:
+ uint32_t driver_region0_start_address_in_kb;
-:299: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#299: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:765:
+ uint32_t used_by_driver_region0_in_kb;
-:300: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#300: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:766:
+ uint32_t reserved32[7];
total: 0 errors, 25 warnings, 20 checks, 281 lines checked
NOTE: For some of the reported defects, checkpatch may be able to
mechanically convert to the typical style using --fix or --fix-inplace.
Your patch has style problems, please review.
NOTE: If any of the errors are false positives, please report
them to the maintainer, see CHECKPATCH in MAINTAINERS.
scripts/checkpatch.pl found problems with your patch.
To disable its check, run your Git command as
(unset GIT_CHECKPATCH; git ...)
$_
Regards,
Luben
On 2022-11-08 09:04, Christian König wrote:
> Yeah, I mean the code looks correct.
>
> It's just that style problems are usually pointed out by automated
> checkers, especially things like dos line endings.
>
> So get that fixed and we can push it immediately.
>
> Thanks,
> Christian.
>
> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>> [AMD Official Use Only - General]
>>
>> + Bokun to help addressing the coding style problem in MKM side.
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Tuesday, November 8, 2022 8:53 PM
>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>>
>> Hi Esther
>>
>> well there are a couple of things which you need to address before getting this merged.
>>
>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>
>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>
>> Then the kernel coding style usually says that with a multi line "if ("
>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>
>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>> But it's correct that this should probably be part of another patch.
>>
>> Regards,
>> Christian.
>>
>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>> [AMD Official Use Only - General]
>>>
>>> Hi @Koenig, Christian,
>>>
>>> Refined as your comment. By the way:
>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>
>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>
>>> Kind regards,
>>> Esther
>>>
>>> -----Original Message-----
>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>> Sent: 2022年11月8日星期二 下午6:33
>>> To: amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
>>> TMR region firstly to make sure TMR can be allocated at 2MB
>>>
>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>> ---
>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> index b81b77a9efa6..032dc2678d7c 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>> }
>>> }
>>>
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t start_addr, fw_size, drv_size;
>>> +
>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>> + drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>> +
>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>> + start_addr,
>>> + fw_size,
>>> + drv_size);
>>> +
>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + /* Use the default scratch size */
>>> + *usage_bytes = 0;
>>> + } else {
>>> + *usage_bytes = drv_size << 10;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>> +
>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>> +
>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>> + drv_size =
>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>> +
>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>> + fw_start_addr,
>>> + fw_size,
>>> + drv_start_addr,
>>> + drv_size);
>>> +
>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + }
>>> +
>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* driver request VRAM reservation for SR-IOV */
>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>> + }
>>> +
>>> + *usage_bytes = 0;
>>> + return 0;
>>> +}
>>> +
>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>> vram_usagebyfirmware);
>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>> - uint32_t start_addr, size;
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>> uint16_t data_offset;
>>> + uint8_t frev, crev;
>>> int usage_bytes = 0;
>>>
>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>> -
>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>> -
>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> - /* Firmware request VRAM reservation for SR-IOV */
>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> - adev->mman.fw_vram_usage_size = size << 10;
>>> - /* Use the default scratch size */
>>> - usage_bytes = 0;
>>> - } else {
>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>> + if (frev == 2 && crev == 1) {
>>> + firmware_usage_v2_1 =
>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>> + firmware_usage_v2_1,
>>> + &usage_bytes);
>>> + } else if (frev >= 2 && crev >= 2) {
>>> + firmware_usage_v2_2 =
>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>> + firmware_usage_v2_2,
>>> + &usage_bytes);
>>> }
>>> }
>>> +
>>> ctx->scratch_size_bytes = 0;
>>> if (usage_bytes == 0)
>>> usage_bytes = 20 * 1024;
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 585460ab8dfd..4a73cb314086 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>> NULL, &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/*
>>> + * Driver Reservation functions
>>> + */
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * free drv reserved vram if it has been reserved.
>>> + */
>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>> +*adev) {
>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL, NULL);
>>> +}
>>> +
>>> /**
>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>> *
>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>> &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
>>> +driver
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * create bo vram reservation from drv.
>>> + */
>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>> +*adev) {
>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>> +
>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>> +
>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>> + adev->mman.drv_vram_usage_size > vram_size)
>>> + return 0;
>>> +
>>> + return amdgpu_bo_create_kernel_at(adev,
>>> + adev->mman.drv_vram_usage_start_offset,
>>> + adev->mman.drv_vram_usage_size,
>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL);
>>> +}
>>> +
>>> /*
>>> * Memoy training reservation functions
>>> */
>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>>
>>> + /*
>>> + *The reserved vram for driver must be pinned to the specified
>>> + *place on the VRAM, so reserve it early.
>>> + */
>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>> + if (r)
>>> + return r;
>>> +
>>> /*
>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>> &adev->mman.sdma_access_ptr);
>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>
>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index 9120ae80ef52..339838675b11 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>> void *fw_vram_usage_va;
>>>
>>> + /* driver VRAM reservation */
>>> + u64 drv_vram_usage_start_offset;
>>> + u64 drv_vram_usage_size;
>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>> +
>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>> struct amdgpu_bo *sdma_access_bo;
>>> void *sdma_access_ptr;
>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> index ff855cb21d3f..c0f56ae653f0 100644
>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>
>>>
>>> -/*
>>> - ***************************************************************************
>>> - Data Table vram_usagebyfirmware structure
>>> -
>>> **********************************************************************
>>> *****
>>> +/*
>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>> + Host driver would overwrite the table with the following
>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>> + } else {
>>> + there is no VBIOS reservation region
>>> + driver must allocate driver reservation region at top of FB.
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + }
>>> + } else { //( NV1X and after)
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>> + }
>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + } else {
>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>> + }
>>> + }
>>> */
>>>
>>> struct vram_usagebyfirmware_v2_1
>>> {
>>> - struct atom_common_table_header table_header;
>>> - uint32_t start_address_in_kb;
>>> - uint16_t used_by_firmware_in_kb;
>>> - uint16_t used_by_driver_in_kb;
>>> + struct atom_common_table_header table_header;
>>> + uint32_t start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t used_by_driver_in_kb;
>>> };
>>>
>>> +struct vram_usagebyfirmware_v2_2 {
>>> + struct atom_common_table_header table_header;
>>> + uint32_t fw_region_start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t reserved;
>>> + uint32_t driver_region0_start_address_in_kb;
>>> + uint32_t used_by_driver_region0_in_kb;
>>> + uint32_t reserved32[7];
>>> +};
>>>
>>> /*
>>>
>>> **********************************************************************
>>> *****
>>> --
>>> 2.25.1
>
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 16:33 ` Luben Tuikov
@ 2022-11-10 5:25 ` Liu01, Tong (Esther)
2022-11-10 9:49 ` Christian König
2022-11-10 15:10 ` Luben Tuikov
0 siblings, 2 replies; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-10 5:25 UTC (permalink / raw)
To: Tuikov, Luben, Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Sohail, Rashid, Deucher, Alexander,
Quan, Evan, Chen, JingWen (Wayne),
Liu, Monk, Zhang, Hawking
[AMD Official Use Only - General]
Hi @Tuikov, Luben,
Which checkpatch.pl you used? I use the one in /linux/script/checkpatch.pl on branch amd-staging-drm-next. And I only got line length warning as:
root@amd-SYS-7048GR-TR:~/gerrit/esther/linux# ./scripts/checkpatch.pl 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch
WARNING: line length of 114 exceeds 100 columns
#256: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
+ * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
WARNING: line length of 113 exceeds 100 columns
#257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
+ * driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
WARNING: line length of 132 exceeds 100 columns
#262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
WARNING: line length of 102 exceeds 100 columns
#266: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
+ * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
WARNING: line length of 134 exceeds 100 columns
#280: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
WARNING: line length of 104 exceeds 100 columns
#288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
+ * dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
total: 0 errors, 6 warnings, 281 lines checked
NOTE: For some of the reported defects, checkpatch may be able to
mechanically convert to the typical style using --fix or --fix-inplace.
0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch has style problems, please review.
NOTE: If any of the errors are false positives, please report
them to the maintainer, see CHECKPATCH in MAINTAINERS.
It seems our scripts checkpatch.pl are different. Could you please provide us with yours? Then I can check my code in further. Thanks.
Kind regards,
Esther
-----Original Message-----
From: Tuikov, Luben <Luben.Tuikov@amd.com>
Sent: 2022年11月9日星期三 上午12:34
To: Koenig, Christian <Christian.Koenig@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Hi,
Applying this patch to amd-staging-drm-next, with checkpatch.pl enabled, generates the following output. Perhaps those issues should be address and fixed.
$git am ~/patches/tongliu01/\[PATCH]\ drm_amdgpu\:\ add\ vram\ reservation\ logic\ based\ on\ vram_usagebyfirmware_v2_2\ -\ Tong\ Liu01\ \<Tong.Liu01\@amd.com\>\ -\ 2022-11-08\ 0532.eml
Applying: drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
-:10: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#10: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:105:
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
-:13: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#13: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:108:
+ uint32_t start_addr, fw_size, drv_size;
-:20: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#20: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:115:
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
-:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#40: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:135:
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
-:43: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#43: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:138:
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
-:45: WARNING:LONG_LINE: line length of 88 exceeds 81 columns
#45: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:140:
+ fw_start_addr =
+le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
-:48: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#48: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:143:
+ drv_start_addr =
+le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
-:49: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
#49: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:144:
+ drv_size =
+le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
-:51: WARNING:LONG_LINE_STRING: line length of 86 exceeds 81 columns
#51: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:146:
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x
+%dkb\n",
-:52: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#52: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:147:
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
-:57: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
#57: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:152:
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
+0) {
-:64: WARNING:LONG_LINE: line length of 83 exceeds 81 columns
#64: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:159:
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
+0) {
-:85: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u8' over 'uint8_t'
#85: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:178:
+ uint8_t frev, crev;
-:109: WARNING:LONG_LINE: line length of 90 exceeds 81 columns
#109: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:181:
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev,
+&data_offset)) {
-:112: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#112: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:184:
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
-:114: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#114: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:186:
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
-:118: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
#118: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:190:
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
-:120: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#120: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:192:
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
-:149: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
#149: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1595:
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
-:168: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u64' over 'uint64_t'
#168: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1633:
+ uint64_t vram_size = adev->gmc.visible_vram_size;
-:239: WARNING:LONG_LINE_COMMENT: line length of 113 exceeds 81 columns
#239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
-:239: WARNING:BLOCK_COMMENT_STYLE: Block comments use * on subsequent lines
#239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
-:240: WARNING:LONG_LINE_COMMENT: line length of 112 exceeds 81 columns
#240: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
-:245: WARNING:LONG_LINE_COMMENT: line length of 132 exceeds 81 columns
#245: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
+ update start_address_in_kb = total_mem_size_in_kb -
+ used_by_firmware_in_kb; ( total_mem_size_in_kb =
+ reg(CONFIG_MEMSIZE)<<10)
-:248: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
#248: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:718:
+ driver reservation start address = (start_address_in_kb -
+ used_by_driver_in_kb)
-:249: WARNING:LONG_LINE_COMMENT: line length of 102 exceeds 81 columns
#249: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
-:257: WARNING:LONG_LINE_COMMENT: line length of 87 exceeds 81 columns
#257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:727:
+ driver reservation start address = (total_mem_size_in_kb -
+ used_by_driver_in_kb)
-:262: WARNING:LONG_LINE_COMMENT: line length of 99 exceeds 81 columns
#262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:732:
+ VBIOS/UEFIGOP update used_by_firmware_in_kb =
+ atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
-:263: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 81 columns
#263: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
+ update start_address_in_kb = total_mem_size_in_kb -
+ used_by_firmware_in_kb; ( total_mem_size_in_kb =
+ reg(CONFIG_MEMSIZE)<<10 )
-:268: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
#268: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:738:
+ driver reservation start address = (start_address_in_kb -
+ used_by_driver_in_kb)
-:271: WARNING:TYPO_SPELLING: 'dirver' may be misspelled - perhaps 'driver'?
#271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
+ dirver can allocate it reservation any place as long as it does
+ overlap pre-OS FW reservation area
^^^^^^
-:271: WARNING:LONG_LINE_COMMENT: line length of 104 exceeds 81 columns
#271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
+ dirver can allocate it reservation any place as long as it does
+ overlap pre-OS FW reservation area
-:273: WARNING:LONG_LINE_COMMENT: line length of 94 exceeds 81 columns
#273: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:743:
+ driver set driver_region0_start_address_in_kb = driver
+ reservation region start address
-:274: WARNING:LONG_LINE_COMMENT: line length of 97 exceeds 81 columns
#274: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:744:
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and
+ start_address_in_kb to zero
-:275: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
#275: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:745:
+ as the reservation for VF as it doesn’t exist. And Host driver
+ should also
-:276: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
#276: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:746:
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
-:288: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:754:
+ uint32_t start_address_in_kb;
-:289: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#289: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:755:
+ uint16_t used_by_firmware_in_kb;
-:290: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#290: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:756:
+ uint16_t used_by_driver_in_kb;
-:295: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#295: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:761:
+ uint32_t fw_region_start_address_in_kb;
-:296: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#296: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:762:
+ uint16_t used_by_firmware_in_kb;
-:297: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
#297: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:763:
+ uint16_t reserved;
-:298: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#298: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:764:
+ uint32_t driver_region0_start_address_in_kb;
-:299: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#299: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:765:
+ uint32_t used_by_driver_region0_in_kb;
-:300: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
#300: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:766:
+ uint32_t reserved32[7];
total: 0 errors, 25 warnings, 20 checks, 281 lines checked
NOTE: For some of the reported defects, checkpatch may be able to
mechanically convert to the typical style using --fix or --fix-inplace.
Your patch has style problems, please review.
NOTE: If any of the errors are false positives, please report
them to the maintainer, see CHECKPATCH in MAINTAINERS.
scripts/checkpatch.pl found problems with your patch.
To disable its check, run your Git command as (unset GIT_CHECKPATCH; git ...) $_
Regards,
Luben
On 2022-11-08 09:04, Christian König wrote:
> Yeah, I mean the code looks correct.
>
> It's just that style problems are usually pointed out by automated
> checkers, especially things like dos line endings.
>
> So get that fixed and we can push it immediately.
>
> Thanks,
> Christian.
>
> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>> [AMD Official Use Only - General]
>>
>> + Bokun to help addressing the coding style problem in MKM side.
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Tuesday, November 8, 2022 8:53 PM
>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Hi Esther
>>
>> well there are a couple of things which you need to address before getting this merged.
>>
>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>
>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>
>> Then the kernel coding style usually says that with a multi line "if ("
>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>
>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>> But it's correct that this should probably be part of another patch.
>>
>> Regards,
>> Christian.
>>
>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>> [AMD Official Use Only - General]
>>>
>>> Hi @Koenig, Christian,
>>>
>>> Refined as your comment. By the way:
>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>
>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>
>>> Kind regards,
>>> Esther
>>>
>>> -----Original Message-----
>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>> Sent: 2022年11月8日星期二 下午6:33
>>> To: amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>
>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>> ---
>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> index b81b77a9efa6..032dc2678d7c 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>> }
>>> }
>>>
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t start_addr, fw_size, drv_size;
>>> +
>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>> + drv_size =
>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>> +
>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>> + start_addr,
>>> + fw_size,
>>> + drv_size);
>>> +
>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + /* Use the default scratch size */
>>> + *usage_bytes = 0;
>>> + } else {
>>> + *usage_bytes = drv_size << 10;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>> +
>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>> + fw_size =
>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>> +
>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>> + drv_size =
>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>> +
>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>> + fw_start_addr,
>>> + fw_size,
>>> + drv_start_addr,
>>> + drv_size);
>>> +
>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + }
>>> +
>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* driver request VRAM reservation for SR-IOV */
>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>> + }
>>> +
>>> + *usage_bytes = 0;
>>> + return 0;
>>> +}
>>> +
>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>> vram_usagebyfirmware);
>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>> - uint32_t start_addr, size;
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>> uint16_t data_offset;
>>> + uint8_t frev, crev;
>>> int usage_bytes = 0;
>>>
>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>> -
>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>> -
>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> - /* Firmware request VRAM reservation for SR-IOV */
>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> - adev->mman.fw_vram_usage_size = size << 10;
>>> - /* Use the default scratch size */
>>> - usage_bytes = 0;
>>> - } else {
>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>> + if (frev == 2 && crev == 1) {
>>> + firmware_usage_v2_1 =
>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>> + firmware_usage_v2_1,
>>> + &usage_bytes);
>>> + } else if (frev >= 2 && crev >= 2) {
>>> + firmware_usage_v2_2 =
>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>> + firmware_usage_v2_2,
>>> + &usage_bytes);
>>> }
>>> }
>>> +
>>> ctx->scratch_size_bytes = 0;
>>> if (usage_bytes == 0)
>>> usage_bytes = 20 * 1024; diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 585460ab8dfd..4a73cb314086 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>> NULL, &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/*
>>> + * Driver Reservation functions
>>> + */
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * free drv reserved vram if it has been reserved.
>>> + */
>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>> +*adev) {
>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL, NULL);
>>> +}
>>> +
>>> /**
>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>> *
>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>> &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>> +from driver
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * create bo vram reservation from drv.
>>> + */
>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>> +*adev) {
>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>> +
>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>> +
>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>> + adev->mman.drv_vram_usage_size > vram_size)
>>> + return 0;
>>> +
>>> + return amdgpu_bo_create_kernel_at(adev,
>>> + adev->mman.drv_vram_usage_start_offset,
>>> + adev->mman.drv_vram_usage_size,
>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL); }
>>> +
>>> /*
>>> * Memoy training reservation functions
>>> */
>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>>
>>> + /*
>>> + *The reserved vram for driver must be pinned to the specified
>>> + *place on the VRAM, so reserve it early.
>>> + */
>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>> + if (r)
>>> + return r;
>>> +
>>> /*
>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>> &adev->mman.sdma_access_ptr);
>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>
>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index 9120ae80ef52..339838675b11 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>> void *fw_vram_usage_va;
>>>
>>> + /* driver VRAM reservation */
>>> + u64 drv_vram_usage_start_offset;
>>> + u64 drv_vram_usage_size;
>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>> +
>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>> struct amdgpu_bo *sdma_access_bo;
>>> void *sdma_access_ptr;
>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> index ff855cb21d3f..c0f56ae653f0 100644
>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>
>>>
>>> -/*
>>> - ***************************************************************************
>>> - Data Table vram_usagebyfirmware structure
>>> -
>>> ********************************************************************
>>> **
>>> *****
>>> +/*
>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>> + Host driver would overwrite the table with the following
>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>> + } else {
>>> + there is no VBIOS reservation region
>>> + driver must allocate driver reservation region at top of FB.
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + }
>>> + } else { //( NV1X and after)
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>> + }
>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + } else {
>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>> + }
>>> + }
>>> */
>>>
>>> struct vram_usagebyfirmware_v2_1
>>> {
>>> - struct atom_common_table_header table_header;
>>> - uint32_t start_address_in_kb;
>>> - uint16_t used_by_firmware_in_kb;
>>> - uint16_t used_by_driver_in_kb;
>>> + struct atom_common_table_header table_header;
>>> + uint32_t start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t used_by_driver_in_kb;
>>> };
>>>
>>> +struct vram_usagebyfirmware_v2_2 {
>>> + struct atom_common_table_header table_header;
>>> + uint32_t fw_region_start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t reserved;
>>> + uint32_t driver_region0_start_address_in_kb;
>>> + uint32_t used_by_driver_region0_in_kb;
>>> + uint32_t reserved32[7];
>>> +};
>>>
>>> /*
>>>
>>> ********************************************************************
>>> **
>>> *****
>>> --
>>> 2.25.1
>
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 5:25 ` Liu01, Tong (Esther)
@ 2022-11-10 9:49 ` Christian König
2022-11-10 15:10 ` Luben Tuikov
1 sibling, 0 replies; 27+ messages in thread
From: Christian König @ 2022-11-10 9:49 UTC (permalink / raw)
To: Liu01, Tong (Esther),
Tuikov, Luben, Chang, HaiJun, amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Sohail, Rashid, Deucher, Alexander,
Quan, Evan, Chen, JingWen (Wayne),
Liu, Monk, Zhang, Hawking
I'm using the checkpatch.pl version from the upstream Linux kernel, but
as far as I know that should be the same.
Are you sending the patch with "git send-email" ? Maybe the mail client
is mangling white space or something like this, when I try to check the
patch here it gives quite some more warnings.
Regards,
Christian.
Am 10.11.22 um 06:25 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi @Tuikov, Luben,
>
> Which checkpatch.pl you used? I use the one in /linux/script/checkpatch.pl on branch amd-staging-drm-next. And I only got line length warning as:
>
> root@amd-SYS-7048GR-TR:~/gerrit/esther/linux# ./scripts/checkpatch.pl 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch
> WARNING: line length of 114 exceeds 100 columns
> #256: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> + * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> WARNING: line length of 113 exceeds 100 columns
> #257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
> + * driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>
> WARNING: line length of 132 exceeds 100 columns
> #262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
> + * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>
> WARNING: line length of 102 exceeds 100 columns
> #266: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
> + * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>
> WARNING: line length of 134 exceeds 100 columns
> #280: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
> + * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>
> WARNING: line length of 104 exceeds 100 columns
> #288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + * dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>
> total: 0 errors, 6 warnings, 281 lines checked
>
> NOTE: For some of the reported defects, checkpatch may be able to
> mechanically convert to the typical style using --fix or --fix-inplace.
>
> 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch has style problems, please review.
>
> NOTE: If any of the errors are false positives, please report
> them to the maintainer, see CHECKPATCH in MAINTAINERS.
>
> It seems our scripts checkpatch.pl are different. Could you please provide us with yours? Then I can check my code in further. Thanks.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Tuikov, Luben <Luben.Tuikov@amd.com>
> Sent: 2022年11月9日星期三 上午12:34
> To: Koenig, Christian <Christian.Koenig@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Hi,
>
> Applying this patch to amd-staging-drm-next, with checkpatch.pl enabled, generates the following output. Perhaps those issues should be address and fixed.
>
> $git am ~/patches/tongliu01/\[PATCH]\ drm_amdgpu\:\ add\ vram\ reservation\ logic\ based\ on\ vram_usagebyfirmware_v2_2\ -\ Tong\ Liu01\ \<Tong.Liu01\@amd.com\>\ -\ 2022-11-08\ 0532.eml
> Applying: drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
> -:10: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #10: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:105:
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>
> -:13: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #13: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:108:
> + uint32_t start_addr, fw_size, drv_size;
>
> -:20: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #20: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:115:
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + start_addr,
>
> -:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #40: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:135:
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>
> -:43: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #43: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:138:
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>
> -:45: WARNING:LONG_LINE: line length of 88 exceeds 81 columns
> #45: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:140:
> + fw_start_addr =
> +le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>
> -:48: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #48: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:143:
> + drv_start_addr =
> +le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>
> -:49: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
> #49: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:144:
> + drv_size =
> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>
> -:51: WARNING:LONG_LINE_STRING: line length of 86 exceeds 81 columns
> #51: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:146:
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x
> +%dkb\n",
>
> -:52: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #52: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:147:
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + fw_start_addr,
>
> -:57: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
> #57: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:152:
> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
> +0) {
>
> -:64: WARNING:LONG_LINE: line length of 83 exceeds 81 columns
> #64: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:159:
> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
> +0) {
>
> -:85: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u8' over 'uint8_t'
> #85: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:178:
> + uint8_t frev, crev;
>
> -:109: WARNING:LONG_LINE: line length of 90 exceeds 81 columns
> #109: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:181:
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev,
> +&data_offset)) {
>
> -:112: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #112: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:184:
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>
> -:114: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #114: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:186:
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
>
> -:118: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #118: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:190:
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>
> -:120: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #120: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:192:
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
>
> -:149: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #149: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1595:
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
>
> -:168: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u64' over 'uint64_t'
> #168: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1633:
> + uint64_t vram_size = adev->gmc.visible_vram_size;
>
> -:239: WARNING:LONG_LINE_COMMENT: line length of 113 exceeds 81 columns
> #239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> -:239: WARNING:BLOCK_COMMENT_STYLE: Block comments use * on subsequent lines
> #239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> -:240: WARNING:LONG_LINE_COMMENT: line length of 112 exceeds 81 columns
> #240: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>
> -:245: WARNING:LONG_LINE_COMMENT: line length of 132 exceeds 81 columns
> #245: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
> + update start_address_in_kb = total_mem_size_in_kb -
> + used_by_firmware_in_kb; ( total_mem_size_in_kb =
> + reg(CONFIG_MEMSIZE)<<10)
>
> -:248: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
> #248: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:718:
> + driver reservation start address = (start_address_in_kb -
> + used_by_driver_in_kb)
>
> -:249: WARNING:LONG_LINE_COMMENT: line length of 102 exceeds 81 columns
> #249: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>
> -:257: WARNING:LONG_LINE_COMMENT: line length of 87 exceeds 81 columns
> #257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:727:
> + driver reservation start address = (total_mem_size_in_kb -
> + used_by_driver_in_kb)
>
> -:262: WARNING:LONG_LINE_COMMENT: line length of 99 exceeds 81 columns
> #262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:732:
> + VBIOS/UEFIGOP update used_by_firmware_in_kb =
> + atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>
> -:263: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 81 columns
> #263: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
> + update start_address_in_kb = total_mem_size_in_kb -
> + used_by_firmware_in_kb; ( total_mem_size_in_kb =
> + reg(CONFIG_MEMSIZE)<<10 )
>
> -:268: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
> #268: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:738:
> + driver reservation start address = (start_address_in_kb -
> + used_by_driver_in_kb)
>
> -:271: WARNING:TYPO_SPELLING: 'dirver' may be misspelled - perhaps 'driver'?
> #271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + dirver can allocate it reservation any place as long as it does
> + overlap pre-OS FW reservation area
> ^^^^^^
>
> -:271: WARNING:LONG_LINE_COMMENT: line length of 104 exceeds 81 columns
> #271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + dirver can allocate it reservation any place as long as it does
> + overlap pre-OS FW reservation area
>
> -:273: WARNING:LONG_LINE_COMMENT: line length of 94 exceeds 81 columns
> #273: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:743:
> + driver set driver_region0_start_address_in_kb = driver
> + reservation region start address
>
> -:274: WARNING:LONG_LINE_COMMENT: line length of 97 exceeds 81 columns
> #274: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:744:
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and
> + start_address_in_kb to zero
>
> -:275: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
> #275: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:745:
> + as the reservation for VF as it doesn’t exist. And Host driver
> + should also
>
> -:276: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
> #276: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:746:
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>
> -:288: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:754:
> + uint32_t start_address_in_kb;
>
> -:289: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #289: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:755:
> + uint16_t used_by_firmware_in_kb;
>
> -:290: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #290: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:756:
> + uint16_t used_by_driver_in_kb;
>
> -:295: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #295: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:761:
> + uint32_t fw_region_start_address_in_kb;
>
> -:296: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #296: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:762:
> + uint16_t used_by_firmware_in_kb;
>
> -:297: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #297: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:763:
> + uint16_t reserved;
>
> -:298: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #298: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:764:
> + uint32_t driver_region0_start_address_in_kb;
>
> -:299: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #299: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:765:
> + uint32_t used_by_driver_region0_in_kb;
>
> -:300: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #300: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:766:
> + uint32_t reserved32[7];
>
> total: 0 errors, 25 warnings, 20 checks, 281 lines checked
>
> NOTE: For some of the reported defects, checkpatch may be able to
> mechanically convert to the typical style using --fix or --fix-inplace.
>
> Your patch has style problems, please review.
>
> NOTE: If any of the errors are false positives, please report
> them to the maintainer, see CHECKPATCH in MAINTAINERS.
> scripts/checkpatch.pl found problems with your patch.
> To disable its check, run your Git command as (unset GIT_CHECKPATCH; git ...) $_
>
> Regards,
> Luben
>
>
> On 2022-11-08 09:04, Christian König wrote:
>> Yeah, I mean the code looks correct.
>>
>> It's just that style problems are usually pointed out by automated
>> checkers, especially things like dos line endings.
>>
>> So get that fixed and we can push it immediately.
>>
>> Thanks,
>> Christian.
>>
>> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>>> [AMD Official Use Only - General]
>>>
>>> + Bokun to help addressing the coding style problem in MKM side.
>>>
>>> -----Original Message-----
>>> From: Koenig, Christian <Christian.Koenig@amd.com>
>>> Sent: Tuesday, November 8, 2022 8:53 PM
>>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>>> amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Hi Esther
>>>
>>> well there are a couple of things which you need to address before getting this merged.
>>>
>>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>>
>>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>>
>>> Then the kernel coding style usually says that with a multi line "if ("
>>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>>
>>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>>> But it's correct that this should probably be part of another patch.
>>>
>>> Regards,
>>> Christian.
>>>
>>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>>> [AMD Official Use Only - General]
>>>>
>>>> Hi @Koenig, Christian,
>>>>
>>>> Refined as your comment. By the way:
>>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>>
>>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>>
>>>> Kind regards,
>>>> Esther
>>>>
>>>> -----Original Message-----
>>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>>> Sent: 2022年11月8日星期二 下午6:33
>>>> To: amd-gfx@lists.freedesktop.org
>>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>>> vram_usagebyfirmware_v2_2
>>>>
>>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>>
>>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>>> ---
>>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> index b81b77a9efa6..032dc2678d7c 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>>> }
>>>> }
>>>>
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t start_addr, fw_size, drv_size;
>>>> +
>>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>>> + drv_size =
>>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>>> + start_addr,
>>>> + fw_size,
>>>> + drv_size);
>>>> +
>>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + /* Use the default scratch size */
>>>> + *usage_bytes = 0;
>>>> + } else {
>>>> + *usage_bytes = drv_size << 10;
>>>> + }
>>>> + return 0;
>>>> +}
>>>> +
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>>> +
>>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>>> + fw_size =
>>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>>> +
>>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>>> + drv_size =
>>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>>> + fw_start_addr,
>>>> + fw_size,
>>>> + drv_start_addr,
>>>> + drv_size);
>>>> +
>>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + }
>>>> +
>>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* driver request VRAM reservation for SR-IOV */
>>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>>> + }
>>>> +
>>>> + *usage_bytes = 0;
>>>> + return 0;
>>>> +}
>>>> +
>>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>>> vram_usagebyfirmware);
>>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>>> - uint32_t start_addr, size;
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>>> uint16_t data_offset;
>>>> + uint8_t frev, crev;
>>>> int usage_bytes = 0;
>>>>
>>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>>> -
>>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>>> -
>>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> - /* Firmware request VRAM reservation for SR-IOV */
>>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> - adev->mman.fw_vram_usage_size = size << 10;
>>>> - /* Use the default scratch size */
>>>> - usage_bytes = 0;
>>>> - } else {
>>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>>> + if (frev == 2 && crev == 1) {
>>>> + firmware_usage_v2_1 =
>>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>>> + firmware_usage_v2_1,
>>>> + &usage_bytes);
>>>> + } else if (frev >= 2 && crev >= 2) {
>>>> + firmware_usage_v2_2 =
>>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>>> + firmware_usage_v2_2,
>>>> + &usage_bytes);
>>>> }
>>>> }
>>>> +
>>>> ctx->scratch_size_bytes = 0;
>>>> if (usage_bytes == 0)
>>>> usage_bytes = 20 * 1024; diff --git
>>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> index 585460ab8dfd..4a73cb314086 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>>> NULL, &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/*
>>>> + * Driver Reservation functions
>>>> + */
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * free drv reserved vram if it has been reserved.
>>>> + */
>>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>>> +*adev) {
>>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL, NULL);
>>>> +}
>>>> +
>>>> /**
>>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>>> *
>>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>>> &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>>> +from driver
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * create bo vram reservation from drv.
>>>> + */
>>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>>> +*adev) {
>>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>>> +
>>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>>> +
>>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>>> + adev->mman.drv_vram_usage_size > vram_size)
>>>> + return 0;
>>>> +
>>>> + return amdgpu_bo_create_kernel_at(adev,
>>>> + adev->mman.drv_vram_usage_start_offset,
>>>> + adev->mman.drv_vram_usage_size,
>>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL); }
>>>> +
>>>> /*
>>>> * Memoy training reservation functions
>>>> */
>>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>>> return r;
>>>> }
>>>>
>>>> + /*
>>>> + *The reserved vram for driver must be pinned to the specified
>>>> + *place on the VRAM, so reserve it early.
>>>> + */
>>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>>> + if (r)
>>>> + return r;
>>>> +
>>>> /*
>>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>>> &adev->mman.sdma_access_ptr);
>>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>>
>>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> index 9120ae80ef52..339838675b11 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>>> void *fw_vram_usage_va;
>>>>
>>>> + /* driver VRAM reservation */
>>>> + u64 drv_vram_usage_start_offset;
>>>> + u64 drv_vram_usage_size;
>>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>>> +
>>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>>> struct amdgpu_bo *sdma_access_bo;
>>>> void *sdma_access_ptr;
>>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> index ff855cb21d3f..c0f56ae653f0 100644
>>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>>
>>>>
>>>> -/*
>>>> - ***************************************************************************
>>>> - Data Table vram_usagebyfirmware structure
>>>> -
>>>> ********************************************************************
>>>> **
>>>> *****
>>>> +/*
>>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>>> + Host driver would overwrite the table with the following
>>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>>> + } else {
>>>> + there is no VBIOS reservation region
>>>> + driver must allocate driver reservation region at top of FB.
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + }
>>>> + } else { //( NV1X and after)
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>>> + }
>>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + } else {
>>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>>> + }
>>>> + }
>>>> */
>>>>
>>>> struct vram_usagebyfirmware_v2_1
>>>> {
>>>> - struct atom_common_table_header table_header;
>>>> - uint32_t start_address_in_kb;
>>>> - uint16_t used_by_firmware_in_kb;
>>>> - uint16_t used_by_driver_in_kb;
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t used_by_driver_in_kb;
>>>> };
>>>>
>>>> +struct vram_usagebyfirmware_v2_2 {
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t fw_region_start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t reserved;
>>>> + uint32_t driver_region0_start_address_in_kb;
>>>> + uint32_t used_by_driver_region0_in_kb;
>>>> + uint32_t reserved32[7];
>>>> +};
>>>>
>>>> /*
>>>>
>>>> ********************************************************************
>>>> **
>>>> *****
>>>> --
>>>> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 14:04 ` Christian König
2022-11-08 16:33 ` Luben Tuikov
@ 2022-11-10 10:14 ` Liu01, Tong (Esther)
2022-11-10 10:18 ` Christian König
1 sibling, 1 reply; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-10 10:14 UTC (permalink / raw)
To: Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid,
Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang, Hawking
[-- Attachment #1: Type: text/plain, Size: 20026 bytes --]
[AMD Official Use Only - General]
Hi Christian,
Please find the attached patch, thanks!
Kind regards,
Esther
-----Original Message-----
From: Koenig, Christian <Christian.Koenig@amd.com>
Sent: 2022年11月8日星期二 下午10:04
To: Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Yeah, I mean the code looks correct.
It's just that style problems are usually pointed out by automated checkers, especially things like dos line endings.
So get that fixed and we can push it immediately.
Thanks,
Christian.
Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
> [AMD Official Use Only - General]
>
> + Bokun to help addressing the coding style problem in MKM side.
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: Tuesday, November 8, 2022 8:53 PM
> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
> amd-gfx@lists.freedesktop.org
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>;
> Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>;
> Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
> <KevinYang.Wang@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>;
> Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
> vram_usagebyfirmware_v2_2
>
> Hi Esther
>
> well there are a couple of things which you need to address before getting this merged.
>
> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>
> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>
> Then the kernel coding style usually says that with a multi line "if ("
> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>
> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
> But it's correct that this should probably be part of another patch.
>
> Regards,
> Christian.
>
> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>> [AMD Official Use Only - General]
>>
>> Hi @Koenig, Christian,
>>
>> Refined as your comment. By the way:
>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>
>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>
>> Kind regards,
>> Esther
>>
>> -----Original Message-----
>> From: Tong Liu01 <Tong.Liu01@amd.com>
>> Sent: 2022年11月8日星期二 下午6:33
>> To: amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
>> TMR region firstly to make sure TMR can be allocated at 2MB
>>
>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>> ---
>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> index b81b77a9efa6..032dc2678d7c 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>> }
>> }
>>
>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>> + int *usage_bytes)
>> +{
>> + uint32_t start_addr, fw_size, drv_size;
>> +
>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>> + drv_size =
>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>> +
>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>> + start_addr,
>> + fw_size,
>> + drv_size);
>> +
>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>> + /* Use the default scratch size */
>> + *usage_bytes = 0;
>> + } else {
>> + *usage_bytes = drv_size << 10;
>> + }
>> + return 0;
>> +}
>> +
>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>> + int *usage_bytes)
>> +{
>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>> +
>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>> + fw_size =
>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>> +
>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>> + drv_size =
>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>> +
>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>> + fw_start_addr,
>> + fw_size,
>> + drv_start_addr,
>> + drv_size);
>> +
>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>> + }
>> +
>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>> + /* driver request VRAM reservation for SR-IOV */
>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>> + }
>> +
>> + *usage_bytes = 0;
>> + return 0;
>> +}
>> +
>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>> struct atom_context *ctx = adev->mode_info.atom_context;
>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>> vram_usagebyfirmware);
>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>> - uint32_t start_addr, size;
>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>> uint16_t data_offset;
>> + uint8_t frev, crev;
>> int usage_bytes = 0;
>>
>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>> -
>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>> -
>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> - /* Firmware request VRAM reservation for SR-IOV */
>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> - adev->mman.fw_vram_usage_size = size << 10;
>> - /* Use the default scratch size */
>> - usage_bytes = 0;
>> - } else {
>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>> + if (frev == 2 && crev == 1) {
>> + firmware_usage_v2_1 =
>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>> + firmware_usage_v2_1,
>> + &usage_bytes);
>> + } else if (frev >= 2 && crev >= 2) {
>> + firmware_usage_v2_2 =
>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>> + firmware_usage_v2_2,
>> + &usage_bytes);
>> }
>> }
>> +
>> ctx->scratch_size_bytes = 0;
>> if (usage_bytes == 0)
>> usage_bytes = 20 * 1024; diff --git
>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> index 585460ab8dfd..4a73cb314086 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>> NULL, &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/*
>> + * Driver Reservation functions
>> + */
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * free drv reserved vram if it has been reserved.
>> + */
>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>> +*adev) {
>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>> + NULL, NULL);
>> +}
>> +
>> /**
>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>> *
>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>> &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>> +from driver
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * create bo vram reservation from drv.
>> + */
>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>> +*adev) {
>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>> +
>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>> +
>> + if (adev->mman.drv_vram_usage_size == 0 ||
>> + adev->mman.drv_vram_usage_size > vram_size)
>> + return 0;
>> +
>> + return amdgpu_bo_create_kernel_at(adev,
>> + adev->mman.drv_vram_usage_start_offset,
>> + adev->mman.drv_vram_usage_size,
>> + AMDGPU_GEM_DOMAIN_VRAM,
>> + &adev->mman.drv_vram_usage_reserved_bo,
>> + NULL); }
>> +
>> /*
>> * Memoy training reservation functions
>> */
>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>> return r;
>> }
>>
>> + /*
>> + *The reserved vram for driver must be pinned to the specified
>> + *place on the VRAM, so reserve it early.
>> + */
>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>> + if (r)
>> + return r;
>> +
>> /*
>> * only NAVI10 and onwards ASIC support for IP discovery.
>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>> &adev->mman.sdma_access_ptr);
>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>
>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> index 9120ae80ef52..339838675b11 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>> void *fw_vram_usage_va;
>>
>> + /* driver VRAM reservation */
>> + u64 drv_vram_usage_start_offset;
>> + u64 drv_vram_usage_size;
>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>> +
>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>> struct amdgpu_bo *sdma_access_bo;
>> void *sdma_access_ptr;
>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>> index ff855cb21d3f..c0f56ae653f0 100644
>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>
>>
>> -/*
>> - ***************************************************************************
>> - Data Table vram_usagebyfirmware structure
>> -
>> *********************************************************************
>> *
>> *****
>> +/*
>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>> + driver can allocate driver reservation region under firmware reservation,
>> + used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>> + Host driver would overwrite the table with the following
>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>> + } else {
>> + there is no VBIOS reservation region
>> + driver must allocate driver reservation region at top of FB.
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + }
>> + } else { //( NV1X and after)
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>> + }
>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>> + driver can allocate driver reservation region under firmware reservation,
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + } else {
>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>> + driver set used_by_driver_region0_in_kb = driver reservation size
>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>> + }
>> + }
>> */
>>
>> struct vram_usagebyfirmware_v2_1
>> {
>> - struct atom_common_table_header table_header;
>> - uint32_t start_address_in_kb;
>> - uint16_t used_by_firmware_in_kb;
>> - uint16_t used_by_driver_in_kb;
>> + struct atom_common_table_header table_header;
>> + uint32_t start_address_in_kb;
>> + uint16_t used_by_firmware_in_kb;
>> + uint16_t used_by_driver_in_kb;
>> };
>>
>> +struct vram_usagebyfirmware_v2_2 {
>> + struct atom_common_table_header table_header;
>> + uint32_t fw_region_start_address_in_kb;
>> + uint16_t used_by_firmware_in_kb;
>> + uint16_t reserved;
>> + uint32_t driver_region0_start_address_in_kb;
>> + uint32_t used_by_driver_region0_in_kb;
>> + uint32_t reserved32[7];
>> +};
>>
>> /*
>>
>> *********************************************************************
>> *
>> *****
>> --
>> 2.25.1
[-- Attachment #2: 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch --]
[-- Type: application/octet-stream, Size: 12474 bytes --]
From 32046009121aeab10ea539b38252ca85c0211c3c Mon Sep 17 00:00:00 2001
From: Tong Liu01 <Tong.Liu01@amd.com>
Date: Thu, 10 Nov 2022 17:31:36 +0800
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to
reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 104 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 63 +++++++++--
4 files changed, 191 insertions(+), 32 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..9b97fa39d47a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,97 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *fw_usage, int *usage_bytes)
+{
+ uint32_t start_addr, fw_size, drv_size;
+
+ start_addr = le32_to_cpu(fw_usage->start_address_in_kb);
+ fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb);
+ drv_size = le16_to_cpu(fw_usage->used_by_driver_in_kb);
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
+ fw_size,
+ drv_size);
+
+ if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes = drv_size << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *fw_usage, int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ fw_start_addr = le32_to_cpu(fw_usage->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(fw_usage->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(fw_usage->used_by_driver_region0_in_kb);
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
+ fw_size,
+ drv_start_addr,
+ drv_size);
+
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *fw_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *fw_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ fw_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ fw_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ fw_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ fw_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 76a8ebfc9e71..b2a9ed3a4583 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1579,6 +1579,23 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL,
+ NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1605,6 +1622,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1772,6 +1814,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1897,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 6a70818039dd..7c38843f411e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -84,6 +84,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..bbe1337a8cee 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,65 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
-*/
+/*
+ * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write
+ * access that region. driver can allocate their own reservation region as long as it does not
+ * overlap firwmare's reservation region.
+ * if (pre-NV1X) atom data table firmwareInfoTable version < 3.3:
+ * in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ * if VBIOS/UEFI GOP is posted:
+ * VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ * driver can allocate driver reservation region under firmware reservation,
+ * used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by
+ * host driver. Host driver would overwrite the table with the following
+ * used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ * set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ * else there is no VBIOS reservation region:
+ * driver must allocate driver reservation region at top of FB.
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ * same as Comment1
+ * else (NV1X and after):
+ * if VBIOS/UEFI GOP is posted:
+ * VBIOS/UEFIGOP update:
+ * used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ * start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * (total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ * if vram_usagebyfirmwareTable version <= 2.1:
+ * driver can allocate driver reservation region under firmware reservation,
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = start_address_in_kb - used_by_driver_in_kb
+ * same as Comment1
+ * else driver can:
+ * allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ * set used_by_driver_region0_in_kb = driver reservation size
+ * set driver_region0_start_address_in_kb = driver reservation region start address
+ * Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to
+ * zero as the reservation for VF as it doesn’t exist. And Host driver should also
+ * update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ */
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 10:14 ` Liu01, Tong (Esther)
@ 2022-11-10 10:18 ` Christian König
2022-11-10 10:47 ` Liu01, Tong (Esther)
0 siblings, 1 reply; 27+ messages in thread
From: Christian König @ 2022-11-10 10:18 UTC (permalink / raw)
To: Liu01, Tong (Esther), Chang, HaiJun, amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid, Deucher,
Alexander, Quan, Evan, Liu, Monk, Zhang, Hawking
Feel free to add my Acked-by: Christian König <christian.koenig@amd.com>
to the patch.
Luben might have some additional comments, but in general I think the
biggest problem here is the mail settings.
Somehow either the mail client or the mail server are corrupting the patch.
Regards,
Christian.
Am 10.11.22 um 11:14 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi Christian,
>
> Please find the attached patch, thanks!
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: 2022年11月8日星期二 下午10:04
> To: Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Yeah, I mean the code looks correct.
>
> It's just that style problems are usually pointed out by automated checkers, especially things like dos line endings.
>
> So get that fixed and we can push it immediately.
>
> Thanks,
> Christian.
>
> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>> [AMD Official Use Only - General]
>>
>> + Bokun to help addressing the coding style problem in MKM side.
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Tuesday, November 8, 2022 8:53 PM
>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>;
>> Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>;
>> Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>> <KevinYang.Wang@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>;
>> Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Hi Esther
>>
>> well there are a couple of things which you need to address before getting this merged.
>>
>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>
>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>
>> Then the kernel coding style usually says that with a multi line "if ("
>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>
>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>> But it's correct that this should probably be part of another patch.
>>
>> Regards,
>> Christian.
>>
>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>> [AMD Official Use Only - General]
>>>
>>> Hi @Koenig, Christian,
>>>
>>> Refined as your comment. By the way:
>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>
>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>
>>> Kind regards,
>>> Esther
>>>
>>> -----Original Message-----
>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>> Sent: 2022年11月8日星期二 下午6:33
>>> To: amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
>>> TMR region firstly to make sure TMR can be allocated at 2MB
>>>
>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>> ---
>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> index b81b77a9efa6..032dc2678d7c 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>> }
>>> }
>>>
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t start_addr, fw_size, drv_size;
>>> +
>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>> + drv_size =
>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>> +
>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>> + start_addr,
>>> + fw_size,
>>> + drv_size);
>>> +
>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + /* Use the default scratch size */
>>> + *usage_bytes = 0;
>>> + } else {
>>> + *usage_bytes = drv_size << 10;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>> +
>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>> + fw_size =
>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>> +
>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>> + drv_size =
>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>> +
>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>> + fw_start_addr,
>>> + fw_size,
>>> + drv_start_addr,
>>> + drv_size);
>>> +
>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + }
>>> +
>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* driver request VRAM reservation for SR-IOV */
>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>> + }
>>> +
>>> + *usage_bytes = 0;
>>> + return 0;
>>> +}
>>> +
>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>> vram_usagebyfirmware);
>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>> - uint32_t start_addr, size;
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>> uint16_t data_offset;
>>> + uint8_t frev, crev;
>>> int usage_bytes = 0;
>>>
>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>> -
>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>> -
>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> - /* Firmware request VRAM reservation for SR-IOV */
>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> - adev->mman.fw_vram_usage_size = size << 10;
>>> - /* Use the default scratch size */
>>> - usage_bytes = 0;
>>> - } else {
>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>> + if (frev == 2 && crev == 1) {
>>> + firmware_usage_v2_1 =
>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>> + firmware_usage_v2_1,
>>> + &usage_bytes);
>>> + } else if (frev >= 2 && crev >= 2) {
>>> + firmware_usage_v2_2 =
>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>> + firmware_usage_v2_2,
>>> + &usage_bytes);
>>> }
>>> }
>>> +
>>> ctx->scratch_size_bytes = 0;
>>> if (usage_bytes == 0)
>>> usage_bytes = 20 * 1024; diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 585460ab8dfd..4a73cb314086 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>> NULL, &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/*
>>> + * Driver Reservation functions
>>> + */
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * free drv reserved vram if it has been reserved.
>>> + */
>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>> +*adev) {
>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL, NULL);
>>> +}
>>> +
>>> /**
>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>> *
>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>> &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>> +from driver
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * create bo vram reservation from drv.
>>> + */
>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>> +*adev) {
>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>> +
>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>> +
>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>> + adev->mman.drv_vram_usage_size > vram_size)
>>> + return 0;
>>> +
>>> + return amdgpu_bo_create_kernel_at(adev,
>>> + adev->mman.drv_vram_usage_start_offset,
>>> + adev->mman.drv_vram_usage_size,
>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL); }
>>> +
>>> /*
>>> * Memoy training reservation functions
>>> */
>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>>
>>> + /*
>>> + *The reserved vram for driver must be pinned to the specified
>>> + *place on the VRAM, so reserve it early.
>>> + */
>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>> + if (r)
>>> + return r;
>>> +
>>> /*
>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>> &adev->mman.sdma_access_ptr);
>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>
>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index 9120ae80ef52..339838675b11 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>> void *fw_vram_usage_va;
>>>
>>> + /* driver VRAM reservation */
>>> + u64 drv_vram_usage_start_offset;
>>> + u64 drv_vram_usage_size;
>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>> +
>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>> struct amdgpu_bo *sdma_access_bo;
>>> void *sdma_access_ptr;
>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> index ff855cb21d3f..c0f56ae653f0 100644
>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>
>>>
>>> -/*
>>> - ***************************************************************************
>>> - Data Table vram_usagebyfirmware structure
>>> -
>>> *********************************************************************
>>> *
>>> *****
>>> +/*
>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>> + Host driver would overwrite the table with the following
>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>> + } else {
>>> + there is no VBIOS reservation region
>>> + driver must allocate driver reservation region at top of FB.
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + }
>>> + } else { //( NV1X and after)
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>> + }
>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + } else {
>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>> + }
>>> + }
>>> */
>>>
>>> struct vram_usagebyfirmware_v2_1
>>> {
>>> - struct atom_common_table_header table_header;
>>> - uint32_t start_address_in_kb;
>>> - uint16_t used_by_firmware_in_kb;
>>> - uint16_t used_by_driver_in_kb;
>>> + struct atom_common_table_header table_header;
>>> + uint32_t start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t used_by_driver_in_kb;
>>> };
>>>
>>> +struct vram_usagebyfirmware_v2_2 {
>>> + struct atom_common_table_header table_header;
>>> + uint32_t fw_region_start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t reserved;
>>> + uint32_t driver_region0_start_address_in_kb;
>>> + uint32_t used_by_driver_region0_in_kb;
>>> + uint32_t reserved32[7];
>>> +};
>>>
>>> /*
>>>
>>> *********************************************************************
>>> *
>>> *****
>>> --
>>> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 10:18 ` Christian König
@ 2022-11-10 10:47 ` Liu01, Tong (Esther)
2022-11-10 10:49 ` Christian König
2022-11-10 12:17 ` Lazar, Lijo
0 siblings, 2 replies; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-10 10:47 UTC (permalink / raw)
To: Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun, Deucher,
Alexander
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid, Quan,
Evan, Liu, Monk, Zhang, Hawking
[-- Attachment #1: Type: text/plain, Size: 21916 bytes --]
[AMD Official Use Only - General]
Hi @Deucher, Alexander & @Koenig, Christian,
Can you give me a Reviewed by:? Now the CI job must has Reviewed by then the patch can be passed. Patch is in the attachment. Thanks a lot.
Kind regards,
Esther
-----Original Message-----
From: Koenig, Christian <Christian.Koenig@amd.com>
Sent: 2022年11月10日星期四 下午6:18
To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Feel free to add my Acked-by: Christian König <christian.koenig@amd.com> to the patch.
Luben might have some additional comments, but in general I think the biggest problem here is the mail settings.
Somehow either the mail client or the mail server are corrupting the patch.
Regards,
Christian.
Am 10.11.22 um 11:14 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi Christian,
>
> Please find the attached patch, thanks!
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: 2022年11月8日星期二 下午10:04
> To: Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther)
> <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun
> <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>;
> Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>;
> Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
> <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
> vram_usagebyfirmware_v2_2
>
> Yeah, I mean the code looks correct.
>
> It's just that style problems are usually pointed out by automated checkers, especially things like dos line endings.
>
> So get that fixed and we can push it immediately.
>
> Thanks,
> Christian.
>
> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>> [AMD Official Use Only - General]
>>
>> + Bokun to help addressing the coding style problem in MKM side.
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Tuesday, November 8, 2022 8:53 PM
>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Hi Esther
>>
>> well there are a couple of things which you need to address before getting this merged.
>>
>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>
>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>
>> Then the kernel coding style usually says that with a multi line "if ("
>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>
>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>> But it's correct that this should probably be part of another patch.
>>
>> Regards,
>> Christian.
>>
>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>> [AMD Official Use Only - General]
>>>
>>> Hi @Koenig, Christian,
>>>
>>> Refined as your comment. By the way:
>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>
>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>
>>> Kind regards,
>>> Esther
>>>
>>> -----Original Message-----
>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>> Sent: 2022年11月8日星期二 下午6:33
>>> To: amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>
>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>> ---
>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> index b81b77a9efa6..032dc2678d7c 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>> }
>>> }
>>>
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t start_addr, fw_size, drv_size;
>>> +
>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>> + drv_size =
>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>> +
>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>> + start_addr,
>>> + fw_size,
>>> + drv_size);
>>> +
>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + /* Use the default scratch size */
>>> + *usage_bytes = 0;
>>> + } else {
>>> + *usage_bytes = drv_size << 10;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>> +
>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>> + fw_size =
>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>> +
>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>> + drv_size =
>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>> +
>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>> + fw_start_addr,
>>> + fw_size,
>>> + drv_start_addr,
>>> + drv_size);
>>> +
>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + }
>>> +
>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* driver request VRAM reservation for SR-IOV */
>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>> + }
>>> +
>>> + *usage_bytes = 0;
>>> + return 0;
>>> +}
>>> +
>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>> vram_usagebyfirmware);
>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>> - uint32_t start_addr, size;
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>> uint16_t data_offset;
>>> + uint8_t frev, crev;
>>> int usage_bytes = 0;
>>>
>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>> -
>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>> -
>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> - /* Firmware request VRAM reservation for SR-IOV */
>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> - adev->mman.fw_vram_usage_size = size << 10;
>>> - /* Use the default scratch size */
>>> - usage_bytes = 0;
>>> - } else {
>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>> + if (frev == 2 && crev == 1) {
>>> + firmware_usage_v2_1 =
>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>> + firmware_usage_v2_1,
>>> + &usage_bytes);
>>> + } else if (frev >= 2 && crev >= 2) {
>>> + firmware_usage_v2_2 =
>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>> + firmware_usage_v2_2,
>>> + &usage_bytes);
>>> }
>>> }
>>> +
>>> ctx->scratch_size_bytes = 0;
>>> if (usage_bytes == 0)
>>> usage_bytes = 20 * 1024; diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 585460ab8dfd..4a73cb314086 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>> NULL, &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/*
>>> + * Driver Reservation functions
>>> + */
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * free drv reserved vram if it has been reserved.
>>> + */
>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>> +*adev) {
>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL, NULL);
>>> +}
>>> +
>>> /**
>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>> *
>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>> &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>> +from driver
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * create bo vram reservation from drv.
>>> + */
>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>> +*adev) {
>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>> +
>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>> +
>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>> + adev->mman.drv_vram_usage_size > vram_size)
>>> + return 0;
>>> +
>>> + return amdgpu_bo_create_kernel_at(adev,
>>> + adev->mman.drv_vram_usage_start_offset,
>>> + adev->mman.drv_vram_usage_size,
>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL); }
>>> +
>>> /*
>>> * Memoy training reservation functions
>>> */
>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>>
>>> + /*
>>> + *The reserved vram for driver must be pinned to the specified
>>> + *place on the VRAM, so reserve it early.
>>> + */
>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>> + if (r)
>>> + return r;
>>> +
>>> /*
>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>> &adev->mman.sdma_access_ptr);
>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>
>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index 9120ae80ef52..339838675b11 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>> void *fw_vram_usage_va;
>>>
>>> + /* driver VRAM reservation */
>>> + u64 drv_vram_usage_start_offset;
>>> + u64 drv_vram_usage_size;
>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>> +
>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>> struct amdgpu_bo *sdma_access_bo;
>>> void *sdma_access_ptr;
>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> index ff855cb21d3f..c0f56ae653f0 100644
>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>
>>>
>>> -/*
>>> - ***************************************************************************
>>> - Data Table vram_usagebyfirmware structure
>>> -
>>> ********************************************************************
>>> *
>>> *
>>> *****
>>> +/*
>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>> + Host driver would overwrite the table with the following
>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>> + } else {
>>> + there is no VBIOS reservation region
>>> + driver must allocate driver reservation region at top of FB.
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + }
>>> + } else { //( NV1X and after)
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>> + }
>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + } else {
>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>> + }
>>> + }
>>> */
>>>
>>> struct vram_usagebyfirmware_v2_1
>>> {
>>> - struct atom_common_table_header table_header;
>>> - uint32_t start_address_in_kb;
>>> - uint16_t used_by_firmware_in_kb;
>>> - uint16_t used_by_driver_in_kb;
>>> + struct atom_common_table_header table_header;
>>> + uint32_t start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t used_by_driver_in_kb;
>>> };
>>>
>>> +struct vram_usagebyfirmware_v2_2 {
>>> + struct atom_common_table_header table_header;
>>> + uint32_t fw_region_start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t reserved;
>>> + uint32_t driver_region0_start_address_in_kb;
>>> + uint32_t used_by_driver_region0_in_kb;
>>> + uint32_t reserved32[7];
>>> +};
>>>
>>> /*
>>>
>>> ********************************************************************
>>> *
>>> *
>>> *****
>>> --
>>> 2.25.1
[-- Attachment #2: 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch --]
[-- Type: application/octet-stream, Size: 12474 bytes --]
From 32046009121aeab10ea539b38252ca85c0211c3c Mon Sep 17 00:00:00 2001
From: Tong Liu01 <Tong.Liu01@amd.com>
Date: Thu, 10 Nov 2022 17:31:36 +0800
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to
reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 104 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 63 +++++++++--
4 files changed, 191 insertions(+), 32 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..9b97fa39d47a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,97 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *fw_usage, int *usage_bytes)
+{
+ uint32_t start_addr, fw_size, drv_size;
+
+ start_addr = le32_to_cpu(fw_usage->start_address_in_kb);
+ fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb);
+ drv_size = le16_to_cpu(fw_usage->used_by_driver_in_kb);
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
+ fw_size,
+ drv_size);
+
+ if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes = drv_size << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *fw_usage, int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ fw_start_addr = le32_to_cpu(fw_usage->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(fw_usage->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(fw_usage->used_by_driver_region0_in_kb);
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
+ fw_size,
+ drv_start_addr,
+ drv_size);
+
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *fw_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *fw_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ fw_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ fw_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ fw_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ fw_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 76a8ebfc9e71..b2a9ed3a4583 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1579,6 +1579,23 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL,
+ NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1605,6 +1622,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1772,6 +1814,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1897,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 6a70818039dd..7c38843f411e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -84,6 +84,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..bbe1337a8cee 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,65 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
-*/
+/*
+ * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write
+ * access that region. driver can allocate their own reservation region as long as it does not
+ * overlap firwmare's reservation region.
+ * if (pre-NV1X) atom data table firmwareInfoTable version < 3.3:
+ * in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ * if VBIOS/UEFI GOP is posted:
+ * VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ * driver can allocate driver reservation region under firmware reservation,
+ * used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by
+ * host driver. Host driver would overwrite the table with the following
+ * used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ * set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ * else there is no VBIOS reservation region:
+ * driver must allocate driver reservation region at top of FB.
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ * same as Comment1
+ * else (NV1X and after):
+ * if VBIOS/UEFI GOP is posted:
+ * VBIOS/UEFIGOP update:
+ * used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ * start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * (total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ * if vram_usagebyfirmwareTable version <= 2.1:
+ * driver can allocate driver reservation region under firmware reservation,
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = start_address_in_kb - used_by_driver_in_kb
+ * same as Comment1
+ * else driver can:
+ * allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ * set used_by_driver_region0_in_kb = driver reservation size
+ * set driver_region0_start_address_in_kb = driver reservation region start address
+ * Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to
+ * zero as the reservation for VF as it doesn’t exist. And Host driver should also
+ * update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ */
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 10:47 ` Liu01, Tong (Esther)
@ 2022-11-10 10:49 ` Christian König
2022-11-10 12:17 ` Lazar, Lijo
1 sibling, 0 replies; 27+ messages in thread
From: Christian König @ 2022-11-10 10:49 UTC (permalink / raw)
To: Liu01, Tong (Esther),
Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun, Deucher,
Alexander
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid, Quan,
Evan, Liu, Monk, Zhang, Hawking
My Acked-by should be sufficient as well.
For a Reviewed-by I would need to have more background on the atombios
tables. Alex is the expert on that.
Regards,
Christian.
Am 10.11.22 um 11:47 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi @Deucher, Alexander & @Koenig, Christian,
>
> Can you give me a Reviewed by:? Now the CI job must has Reviewed by then the patch can be passed. Patch is in the attachment. Thanks a lot.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: 2022年11月10日星期四 下午6:18
> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Feel free to add my Acked-by: Christian König <christian.koenig@amd.com> to the patch.
>
> Luben might have some additional comments, but in general I think the biggest problem here is the mail settings.
>
> Somehow either the mail client or the mail server are corrupting the patch.
>
> Regards,
> Christian.
>
> Am 10.11.22 um 11:14 schrieb Liu01, Tong (Esther):
>> [AMD Official Use Only - General]
>>
>> Hi Christian,
>>
>> Please find the attached patch, thanks!
>>
>> Kind regards,
>> Esther
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: 2022年11月8日星期二 下午10:04
>> To: Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther)
>> <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun
>> <Bokun.Zhang@amd.com>
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>;
>> Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>;
>> Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>> <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Yeah, I mean the code looks correct.
>>
>> It's just that style problems are usually pointed out by automated checkers, especially things like dos line endings.
>>
>> So get that fixed and we can push it immediately.
>>
>> Thanks,
>> Christian.
>>
>> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>>> [AMD Official Use Only - General]
>>>
>>> + Bokun to help addressing the coding style problem in MKM side.
>>>
>>> -----Original Message-----
>>> From: Koenig, Christian <Christian.Koenig@amd.com>
>>> Sent: Tuesday, November 8, 2022 8:53 PM
>>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>>> amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Hi Esther
>>>
>>> well there are a couple of things which you need to address before getting this merged.
>>>
>>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>>
>>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>>
>>> Then the kernel coding style usually says that with a multi line "if ("
>>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>>
>>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>>> But it's correct that this should probably be part of another patch.
>>>
>>> Regards,
>>> Christian.
>>>
>>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>>> [AMD Official Use Only - General]
>>>>
>>>> Hi @Koenig, Christian,
>>>>
>>>> Refined as your comment. By the way:
>>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>>
>>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>>
>>>> Kind regards,
>>>> Esther
>>>>
>>>> -----Original Message-----
>>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>>> Sent: 2022年11月8日星期二 下午6:33
>>>> To: amd-gfx@lists.freedesktop.org
>>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>>> vram_usagebyfirmware_v2_2
>>>>
>>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>>
>>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>>> ---
>>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> index b81b77a9efa6..032dc2678d7c 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>>> }
>>>> }
>>>>
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t start_addr, fw_size, drv_size;
>>>> +
>>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>>> + drv_size =
>>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>>> + start_addr,
>>>> + fw_size,
>>>> + drv_size);
>>>> +
>>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + /* Use the default scratch size */
>>>> + *usage_bytes = 0;
>>>> + } else {
>>>> + *usage_bytes = drv_size << 10;
>>>> + }
>>>> + return 0;
>>>> +}
>>>> +
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>>> +
>>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>>> + fw_size =
>>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>>> +
>>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>>> + drv_size =
>>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>>> + fw_start_addr,
>>>> + fw_size,
>>>> + drv_start_addr,
>>>> + drv_size);
>>>> +
>>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + }
>>>> +
>>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* driver request VRAM reservation for SR-IOV */
>>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>>> + }
>>>> +
>>>> + *usage_bytes = 0;
>>>> + return 0;
>>>> +}
>>>> +
>>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>>> vram_usagebyfirmware);
>>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>>> - uint32_t start_addr, size;
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>>> uint16_t data_offset;
>>>> + uint8_t frev, crev;
>>>> int usage_bytes = 0;
>>>>
>>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>>> -
>>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>>> -
>>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> - /* Firmware request VRAM reservation for SR-IOV */
>>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> - adev->mman.fw_vram_usage_size = size << 10;
>>>> - /* Use the default scratch size */
>>>> - usage_bytes = 0;
>>>> - } else {
>>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>>> + if (frev == 2 && crev == 1) {
>>>> + firmware_usage_v2_1 =
>>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>>> + firmware_usage_v2_1,
>>>> + &usage_bytes);
>>>> + } else if (frev >= 2 && crev >= 2) {
>>>> + firmware_usage_v2_2 =
>>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>>> + firmware_usage_v2_2,
>>>> + &usage_bytes);
>>>> }
>>>> }
>>>> +
>>>> ctx->scratch_size_bytes = 0;
>>>> if (usage_bytes == 0)
>>>> usage_bytes = 20 * 1024; diff --git
>>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> index 585460ab8dfd..4a73cb314086 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>>> NULL, &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/*
>>>> + * Driver Reservation functions
>>>> + */
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * free drv reserved vram if it has been reserved.
>>>> + */
>>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>>> +*adev) {
>>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL, NULL);
>>>> +}
>>>> +
>>>> /**
>>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>>> *
>>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>>> &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>>> +from driver
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * create bo vram reservation from drv.
>>>> + */
>>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>>> +*adev) {
>>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>>> +
>>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>>> +
>>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>>> + adev->mman.drv_vram_usage_size > vram_size)
>>>> + return 0;
>>>> +
>>>> + return amdgpu_bo_create_kernel_at(adev,
>>>> + adev->mman.drv_vram_usage_start_offset,
>>>> + adev->mman.drv_vram_usage_size,
>>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL); }
>>>> +
>>>> /*
>>>> * Memoy training reservation functions
>>>> */
>>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>>> return r;
>>>> }
>>>>
>>>> + /*
>>>> + *The reserved vram for driver must be pinned to the specified
>>>> + *place on the VRAM, so reserve it early.
>>>> + */
>>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>>> + if (r)
>>>> + return r;
>>>> +
>>>> /*
>>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>>> &adev->mman.sdma_access_ptr);
>>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>>
>>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> index 9120ae80ef52..339838675b11 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>>> void *fw_vram_usage_va;
>>>>
>>>> + /* driver VRAM reservation */
>>>> + u64 drv_vram_usage_start_offset;
>>>> + u64 drv_vram_usage_size;
>>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>>> +
>>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>>> struct amdgpu_bo *sdma_access_bo;
>>>> void *sdma_access_ptr;
>>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> index ff855cb21d3f..c0f56ae653f0 100644
>>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>>
>>>>
>>>> -/*
>>>> - ***************************************************************************
>>>> - Data Table vram_usagebyfirmware structure
>>>> -
>>>> ********************************************************************
>>>> *
>>>> *
>>>> *****
>>>> +/*
>>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>>> + Host driver would overwrite the table with the following
>>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>>> + } else {
>>>> + there is no VBIOS reservation region
>>>> + driver must allocate driver reservation region at top of FB.
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + }
>>>> + } else { //( NV1X and after)
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>>> + }
>>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + } else {
>>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>>> + }
>>>> + }
>>>> */
>>>>
>>>> struct vram_usagebyfirmware_v2_1
>>>> {
>>>> - struct atom_common_table_header table_header;
>>>> - uint32_t start_address_in_kb;
>>>> - uint16_t used_by_firmware_in_kb;
>>>> - uint16_t used_by_driver_in_kb;
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t used_by_driver_in_kb;
>>>> };
>>>>
>>>> +struct vram_usagebyfirmware_v2_2 {
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t fw_region_start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t reserved;
>>>> + uint32_t driver_region0_start_address_in_kb;
>>>> + uint32_t used_by_driver_region0_in_kb;
>>>> + uint32_t reserved32[7];
>>>> +};
>>>>
>>>> /*
>>>>
>>>> ********************************************************************
>>>> *
>>>> *
>>>> *****
>>>> --
>>>> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 10:47 ` Liu01, Tong (Esther)
2022-11-10 10:49 ` Christian König
@ 2022-11-10 12:17 ` Lazar, Lijo
1 sibling, 0 replies; 27+ messages in thread
From: Lazar, Lijo @ 2022-11-10 12:17 UTC (permalink / raw)
To: Liu01, Tong (Esther),
Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun, Deucher,
Alexander
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Sohail, Rashid, Quan,
Evan, Liu, Monk, Zhang, Hawking
[AMD Official Use Only - General]
For consistency, you may avoid magic numbers and change this kind of lines to
if ((start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
to
if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) !=
(uint32_t)( ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION <<
ATOM_VRAM_OPERATION_FLAGS_SHIFT))
Thanks,
Lijo
-----Original Message-----
From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Liu01, Tong (Esther)
Sent: Thursday, November 10, 2022 4:17 PM
To: Koenig, Christian <Christian.Koenig@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>
Cc: Xiao, Jack <Jack.Xiao@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
Subject: RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
[AMD Official Use Only - General]
Hi @Deucher, Alexander & @Koenig, Christian,
Can you give me a Reviewed by:? Now the CI job must has Reviewed by then the patch can be passed. Patch is in the attachment. Thanks a lot.
Kind regards,
Esther
-----Original Message-----
From: Koenig, Christian <Christian.Koenig@amd.com>
Sent: 2022年11月10日星期四 下午6:18
To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Feel free to add my Acked-by: Christian König <christian.koenig@amd.com> to the patch.
Luben might have some additional comments, but in general I think the biggest problem here is the mail settings.
Somehow either the mail client or the mail server are corrupting the patch.
Regards,
Christian.
Am 10.11.22 um 11:14 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi Christian,
>
> Please find the attached patch, thanks!
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: 2022年11月8日星期二 下午10:04
> To: Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther)
> <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun
> <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>;
> Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>;
> Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
> <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
> vram_usagebyfirmware_v2_2
>
> Yeah, I mean the code looks correct.
>
> It's just that style problems are usually pointed out by automated checkers, especially things like dos line endings.
>
> So get that fixed and we can push it immediately.
>
> Thanks,
> Christian.
>
> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>> [AMD Official Use Only - General]
>>
>> + Bokun to help addressing the coding style problem in MKM side.
>>
>> -----Original Message-----
>> From: Koenig, Christian <Christian.Koenig@amd.com>
>> Sent: Tuesday, November 8, 2022 8:53 PM
>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>> amd-gfx@lists.freedesktop.org
>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>> vram_usagebyfirmware_v2_2
>>
>> Hi Esther
>>
>> well there are a couple of things which you need to address before getting this merged.
>>
>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>
>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>
>> Then the kernel coding style usually says that with a multi line "if ("
>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>
>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>> But it's correct that this should probably be part of another patch.
>>
>> Regards,
>> Christian.
>>
>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>> [AMD Official Use Only - General]
>>>
>>> Hi @Koenig, Christian,
>>>
>>> Refined as your comment. By the way:
>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>
>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>
>>> Kind regards,
>>> Esther
>>>
>>> -----Original Message-----
>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>> Sent: 2022年11月8日星期二 下午6:33
>>> To: amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>
>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>> ---
>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> index b81b77a9efa6..032dc2678d7c 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>> }
>>> }
>>>
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t start_addr, fw_size, drv_size;
>>> +
>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>> + drv_size =
>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>> +
>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>> + start_addr,
>>> + fw_size,
>>> + drv_size);
>>> +
>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + /* Use the default scratch size */
>>> + *usage_bytes = 0;
>>> + } else {
>>> + *usage_bytes = drv_size << 10;
>>> + }
>>> + return 0;
>>> +}
>>> +
>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>> + int *usage_bytes)
>>> +{
>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>> +
>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>> + fw_size =
>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>> +
>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>> + drv_size =
>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>> +
>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>> + fw_start_addr,
>>> + fw_size,
>>> + drv_start_addr,
>>> + drv_size);
>>> +
>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* Firmware request VRAM reservation for SR-IOV */
>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>> + }
>>> +
>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>> + /* driver request VRAM reservation for SR-IOV */
>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>> + }
>>> +
>>> + *usage_bytes = 0;
>>> + return 0;
>>> +}
>>> +
>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>> vram_usagebyfirmware);
>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>> - uint32_t start_addr, size;
>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>> uint16_t data_offset;
>>> + uint8_t frev, crev;
>>> int usage_bytes = 0;
>>>
>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>> -
>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>> -
>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>> - /* Firmware request VRAM reservation for SR-IOV */
>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>> - adev->mman.fw_vram_usage_size = size << 10;
>>> - /* Use the default scratch size */
>>> - usage_bytes = 0;
>>> - } else {
>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>> + if (frev == 2 && crev == 1) {
>>> + firmware_usage_v2_1 =
>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>> + firmware_usage_v2_1,
>>> + &usage_bytes);
>>> + } else if (frev >= 2 && crev >= 2) {
>>> + firmware_usage_v2_2 =
>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>> + firmware_usage_v2_2,
>>> + &usage_bytes);
>>> }
>>> }
>>> +
>>> ctx->scratch_size_bytes = 0;
>>> if (usage_bytes == 0)
>>> usage_bytes = 20 * 1024; diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 585460ab8dfd..4a73cb314086 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>> NULL, &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/*
>>> + * Driver Reservation functions
>>> + */
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * free drv reserved vram if it has been reserved.
>>> + */
>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>> +*adev) {
>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL, NULL);
>>> +}
>>> +
>>> /**
>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>> *
>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>> &adev->mman.fw_vram_usage_va);
>>> }
>>>
>>> +/**
>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>> +from driver
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * create bo vram reservation from drv.
>>> + */
>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>> +*adev) {
>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>> +
>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>> +
>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>> + adev->mman.drv_vram_usage_size > vram_size)
>>> + return 0;
>>> +
>>> + return amdgpu_bo_create_kernel_at(adev,
>>> + adev->mman.drv_vram_usage_start_offset,
>>> + adev->mman.drv_vram_usage_size,
>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>> + NULL); }
>>> +
>>> /*
>>> * Memoy training reservation functions
>>> */
>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>>
>>> + /*
>>> + *The reserved vram for driver must be pinned to the specified
>>> + *place on the VRAM, so reserve it early.
>>> + */
>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>> + if (r)
>>> + return r;
>>> +
>>> /*
>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>> &adev->mman.sdma_access_ptr);
>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>
>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> index 9120ae80ef52..339838675b11 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>> void *fw_vram_usage_va;
>>>
>>> + /* driver VRAM reservation */
>>> + u64 drv_vram_usage_start_offset;
>>> + u64 drv_vram_usage_size;
>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>> +
>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>> struct amdgpu_bo *sdma_access_bo;
>>> void *sdma_access_ptr;
>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> index ff855cb21d3f..c0f56ae653f0 100644
>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>
>>>
>>> -/*
>>> - ***************************************************************************
>>> - Data Table vram_usagebyfirmware structure
>>> -
>>> ********************************************************************
>>> *
>>> *
>>> *****
>>> +/*
>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>> + Host driver would overwrite the table with the following
>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>> + } else {
>>> + there is no VBIOS reservation region
>>> + driver must allocate driver reservation region at top of FB.
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + }
>>> + } else { //( NV1X and after)
>>> + if( VBIOS/UEFI GOP is posted ) {
>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>> + }
>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>> + driver can allocate driver reservation region under firmware reservation,
>>> + driver set used_by_driver_in_kb = driver reservation size
>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>> + same as Comment1
>>> + } else {
>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>> + }
>>> + }
>>> */
>>>
>>> struct vram_usagebyfirmware_v2_1
>>> {
>>> - struct atom_common_table_header table_header;
>>> - uint32_t start_address_in_kb;
>>> - uint16_t used_by_firmware_in_kb;
>>> - uint16_t used_by_driver_in_kb;
>>> + struct atom_common_table_header table_header;
>>> + uint32_t start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t used_by_driver_in_kb;
>>> };
>>>
>>> +struct vram_usagebyfirmware_v2_2 {
>>> + struct atom_common_table_header table_header;
>>> + uint32_t fw_region_start_address_in_kb;
>>> + uint16_t used_by_firmware_in_kb;
>>> + uint16_t reserved;
>>> + uint32_t driver_region0_start_address_in_kb;
>>> + uint32_t used_by_driver_region0_in_kb;
>>> + uint32_t reserved32[7];
>>> +};
>>>
>>> /*
>>>
>>> ********************************************************************
>>> *
>>> *
>>> *****
>>> --
>>> 2.25.1
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-10 5:25 ` Liu01, Tong (Esther)
2022-11-10 9:49 ` Christian König
@ 2022-11-10 15:10 ` Luben Tuikov
1 sibling, 0 replies; 27+ messages in thread
From: Luben Tuikov @ 2022-11-10 15:10 UTC (permalink / raw)
To: Liu01, Tong (Esther),
Koenig, Christian, Chang, HaiJun, amd-gfx, Zhang, Bokun
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Sohail, Rashid, Deucher, Alexander,
Quan, Evan, Chen, JingWen (Wayne),
Liu, Monk, Zhang, Hawking
Hi,
My settings for checkpatch.pl are:
--strict --emacs --show-types --codespell
Please see the resource I sent in the chat.
Regards,
Luben
On 2022-11-10 00:25, Liu01, Tong (Esther) wrote:
> [AMD Official Use Only - General]
>
> Hi @Tuikov, Luben,
>
> Which checkpatch.pl you used? I use the one in /linux/script/checkpatch.pl on branch amd-staging-drm-next. And I only got line length warning as:
>
> root@amd-SYS-7048GR-TR:~/gerrit/esther/linux# ./scripts/checkpatch.pl 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch
> WARNING: line length of 114 exceeds 100 columns
> #256: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> + * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> WARNING: line length of 113 exceeds 100 columns
> #257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
> + * driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>
> WARNING: line length of 132 exceeds 100 columns
> #262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
> + * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>
> WARNING: line length of 102 exceeds 100 columns
> #266: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
> + * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>
> WARNING: line length of 134 exceeds 100 columns
> #280: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
> + * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>
> WARNING: line length of 104 exceeds 100 columns
> #288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + * dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>
> total: 0 errors, 6 warnings, 281 lines checked
>
> NOTE: For some of the reported defects, checkpatch may be able to
> mechanically convert to the typical style using --fix or --fix-inplace.
>
> 0001-drm-amdgpu-add-vram-reservation-logic-based-on-vram_.patch has style problems, please review.
>
> NOTE: If any of the errors are false positives, please report
> them to the maintainer, see CHECKPATCH in MAINTAINERS.
>
> It seems our scripts checkpatch.pl are different. Could you please provide us with yours? Then I can check my code in further. Thanks.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Tuikov, Luben <Luben.Tuikov@amd.com>
> Sent: 2022年11月9日星期三 上午12:34
> To: Koenig, Christian <Christian.Koenig@amd.com>; Chang, HaiJun <HaiJun.Chang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org; Zhang, Bokun <Bokun.Zhang@amd.com>
> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Hi,
>
> Applying this patch to amd-staging-drm-next, with checkpatch.pl enabled, generates the following output. Perhaps those issues should be address and fixed.
>
> $git am ~/patches/tongliu01/\[PATCH]\ drm_amdgpu\:\ add\ vram\ reservation\ logic\ based\ on\ vram_usagebyfirmware_v2_2\ -\ Tong\ Liu01\ \<Tong.Liu01\@amd.com\>\ -\ 2022-11-08\ 0532.eml
> Applying: drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
> -:10: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #10: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:105:
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>
> -:13: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #13: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:108:
> + uint32_t start_addr, fw_size, drv_size;
>
> -:20: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #20: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:115:
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + start_addr,
>
> -:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #40: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:135:
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>
> -:43: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #43: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:138:
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>
> -:45: WARNING:LONG_LINE: line length of 88 exceeds 81 columns
> #45: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:140:
> + fw_start_addr =
> +le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>
> -:48: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #48: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:143:
> + drv_start_addr =
> +le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>
> -:49: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
> #49: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:144:
> + drv_size =
> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>
> -:51: WARNING:LONG_LINE_STRING: line length of 86 exceeds 81 columns
> #51: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:146:
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x
> +%dkb\n",
>
> -:52: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #52: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:147:
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + fw_start_addr,
>
> -:57: WARNING:LONG_LINE: line length of 82 exceeds 81 columns
> #57: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:152:
> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
> +0) {
>
> -:64: WARNING:LONG_LINE: line length of 83 exceeds 81 columns
> #64: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:159:
> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) ==
> +0) {
>
> -:85: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u8' over 'uint8_t'
> #85: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:178:
> + uint8_t frev, crev;
>
> -:109: WARNING:LONG_LINE: line length of 90 exceeds 81 columns
> #109: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:181:
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev,
> +&data_offset)) {
>
> -:112: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #112: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:184:
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>
> -:114: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #114: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:186:
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
>
> -:118: WARNING:LONG_LINE: line length of 94 exceeds 81 columns
> #118: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:190:
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>
> -:120: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #120: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c:192:
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
>
> -:149: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis
> #149: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1595:
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
>
> -:168: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u64' over 'uint64_t'
> #168: FILE: drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c:1633:
> + uint64_t vram_size = adev->gmc.visible_vram_size;
>
> -:239: WARNING:LONG_LINE_COMMENT: line length of 113 exceeds 81 columns
> #239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> -:239: WARNING:BLOCK_COMMENT_STYLE: Block comments use * on subsequent lines
> #239: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:709:
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>
> -:240: WARNING:LONG_LINE_COMMENT: line length of 112 exceeds 81 columns
> #240: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:710:
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>
> -:245: WARNING:LONG_LINE_COMMENT: line length of 132 exceeds 81 columns
> #245: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:715:
> + update start_address_in_kb = total_mem_size_in_kb -
> + used_by_firmware_in_kb; ( total_mem_size_in_kb =
> + reg(CONFIG_MEMSIZE)<<10)
>
> -:248: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
> #248: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:718:
> + driver reservation start address = (start_address_in_kb -
> + used_by_driver_in_kb)
>
> -:249: WARNING:LONG_LINE_COMMENT: line length of 102 exceeds 81 columns
> #249: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:719:
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>
> -:257: WARNING:LONG_LINE_COMMENT: line length of 87 exceeds 81 columns
> #257: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:727:
> + driver reservation start address = (total_mem_size_in_kb -
> + used_by_driver_in_kb)
>
> -:262: WARNING:LONG_LINE_COMMENT: line length of 99 exceeds 81 columns
> #262: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:732:
> + VBIOS/UEFIGOP update used_by_firmware_in_kb =
> + atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>
> -:263: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 81 columns
> #263: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:733:
> + update start_address_in_kb = total_mem_size_in_kb -
> + used_by_firmware_in_kb; ( total_mem_size_in_kb =
> + reg(CONFIG_MEMSIZE)<<10 )
>
> -:268: WARNING:LONG_LINE_COMMENT: line length of 86 exceeds 81 columns
> #268: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:738:
> + driver reservation start address = (start_address_in_kb -
> + used_by_driver_in_kb)
>
> -:271: WARNING:TYPO_SPELLING: 'dirver' may be misspelled - perhaps 'driver'?
> #271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + dirver can allocate it reservation any place as long as it does
> + overlap pre-OS FW reservation area
> ^^^^^^
>
> -:271: WARNING:LONG_LINE_COMMENT: line length of 104 exceeds 81 columns
> #271: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:741:
> + dirver can allocate it reservation any place as long as it does
> + overlap pre-OS FW reservation area
>
> -:273: WARNING:LONG_LINE_COMMENT: line length of 94 exceeds 81 columns
> #273: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:743:
> + driver set driver_region0_start_address_in_kb = driver
> + reservation region start address
>
> -:274: WARNING:LONG_LINE_COMMENT: line length of 97 exceeds 81 columns
> #274: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:744:
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and
> + start_address_in_kb to zero
>
> -:275: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
> #275: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:745:
> + as the reservation for VF as it doesn’t exist. And Host driver
> + should also
>
> -:276: WARNING:LONG_LINE_COMMENT: line length of 83 exceeds 81 columns
> #276: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:746:
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>
> -:288: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #288: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:754:
> + uint32_t start_address_in_kb;
>
> -:289: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #289: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:755:
> + uint16_t used_by_firmware_in_kb;
>
> -:290: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #290: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:756:
> + uint16_t used_by_driver_in_kb;
>
> -:295: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #295: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:761:
> + uint32_t fw_region_start_address_in_kb;
>
> -:296: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #296: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:762:
> + uint16_t used_by_firmware_in_kb;
>
> -:297: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u16' over 'uint16_t'
> #297: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:763:
> + uint16_t reserved;
>
> -:298: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #298: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:764:
> + uint32_t driver_region0_start_address_in_kb;
>
> -:299: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #299: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:765:
> + uint32_t used_by_driver_region0_in_kb;
>
> -:300: CHECK:PREFER_KERNEL_TYPES: Prefer kernel type 'u32' over 'uint32_t'
> #300: FILE: drivers/gpu/drm/amd/include/atomfirmware.h:766:
> + uint32_t reserved32[7];
>
> total: 0 errors, 25 warnings, 20 checks, 281 lines checked
>
> NOTE: For some of the reported defects, checkpatch may be able to
> mechanically convert to the typical style using --fix or --fix-inplace.
>
> Your patch has style problems, please review.
>
> NOTE: If any of the errors are false positives, please report
> them to the maintainer, see CHECKPATCH in MAINTAINERS.
> scripts/checkpatch.pl found problems with your patch.
> To disable its check, run your Git command as (unset GIT_CHECKPATCH; git ...) $_
>
> Regards,
> Luben
>
>
> On 2022-11-08 09:04, Christian König wrote:
>> Yeah, I mean the code looks correct.
>>
>> It's just that style problems are usually pointed out by automated
>> checkers, especially things like dos line endings.
>>
>> So get that fixed and we can push it immediately.
>>
>> Thanks,
>> Christian.
>>
>> Am 08.11.22 um 14:49 schrieb Chang, HaiJun:
>>> [AMD Official Use Only - General]
>>>
>>> + Bokun to help addressing the coding style problem in MKM side.
>>>
>>> -----Original Message-----
>>> From: Koenig, Christian <Christian.Koenig@amd.com>
>>> Sent: Tuesday, November 8, 2022 8:53 PM
>>> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>;
>>> amd-gfx@lists.freedesktop.org
>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher,
>>> Alexander <Alexander.Deucher@amd.com>; Xiao, Jack
>>> <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu,
>>> Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang,
>>> Yang(Kevin) <KevinYang.Wang@amd.com>; Chang, HaiJun
>>> <HaiJun.Chang@amd.com>; Sohail, Rashid <Rashid.Sohail@amd.com>
>>> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on
>>> vram_usagebyfirmware_v2_2
>>>
>>> Hi Esther
>>>
>>> well there are a couple of things which you need to address before getting this merged.
>>>
>>> First of all the patch you send out uses dos line endings instead of the unix line endings. Not sure how you manage to do that, but please use "git send-email" instead to avoid that.
>>>
>>> Then your patch contains a bunch of white spaces after code warning which checkpatch.pl complains about (after ignoring the dos line ending warnings). So this was clearly not properly checked with checkpatch.pl.
>>>
>>> Then the kernel coding style usually says that with a multi line "if ("
>>> the next lines should start after the opening "(". In other words intend with tabs and the whitespaces. I'm not sure what editor you are using, but there are standard settings available for basically all large editors which does stuff like that automatically. Please try to use one of those.
>>>
>>> Regarding the casing of the values it's a good argument that you only move the code around, but the general coding style is just extremely questionable. The defines should use the lowest necessary integer type.
>>> But it's correct that this should probably be part of another patch.
>>>
>>> Regards,
>>> Christian.
>>>
>>> Am 08.11.22 um 11:40 schrieb Liu01, Tong (Esther):
>>>> [AMD Official Use Only - General]
>>>>
>>>> Hi @Koenig, Christian,
>>>>
>>>> Refined as your comment. By the way:
>>>> if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT))
>>>>
>>>> This part is the old code, I just move it out from the original function to shrink the function size as your comment before. And now I just removed the first uint32_t since if remove both will cause "warning: bitwise comparison always evaluates to false". And I tested the code after removing the first uint32_t, the code works well. Please review the new patch, thanks.
>>>>
>>>> Kind regards,
>>>> Esther
>>>>
>>>> -----Original Message-----
>>>> From: Tong Liu01 <Tong.Liu01@amd.com>
>>>> Sent: 2022年11月8日星期二 下午6:33
>>>> To: amd-gfx@lists.freedesktop.org
>>>> Cc: Quan, Evan <Evan.Quan@amd.com>; Chen, Horace
>>>> <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig,
>>>> Christian <Christian.Koenig@amd.com>; Deucher, Alexander
>>>> <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang,
>>>> Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu,
>>>> Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin)
>>>> <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
>>>> Subject: [PATCH] drm/amdgpu: add vram reservation logic based on
>>>> vram_usagebyfirmware_v2_2
>>>>
>>>> Move TMR region from top of FB to 2MB for FFBM, so we need to
>>>> reserve TMR region firstly to make sure TMR can be allocated at 2MB
>>>>
>>>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>>>> ---
>>>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
>>>> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
>>>> 4 files changed, 192 insertions(+), 31 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> index b81b77a9efa6..032dc2678d7c 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>>>> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
>>>> }
>>>> }
>>>>
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t start_addr, fw_size, drv_size;
>>>> +
>>>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>>>> + fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>>>> + drv_size =
>>>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>>>> + start_addr,
>>>> + fw_size,
>>>> + drv_size);
>>>> +
>>>> + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + /* Use the default scratch size */
>>>> + *usage_bytes = 0;
>>>> + } else {
>>>> + *usage_bytes = drv_size << 10;
>>>> + }
>>>> + return 0;
>>>> +}
>>>> +
>>>> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
>>>> + int *usage_bytes)
>>>> +{
>>>> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
>>>> +
>>>> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
>>>> + fw_size =
>>>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>>>> +
>>>> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
>>>> + drv_size =
>>>> +le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>>>> +
>>>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>>>> + fw_start_addr,
>>>> + fw_size,
>>>> + drv_start_addr,
>>>> + drv_size);
>>>> +
>>>> + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* Firmware request VRAM reservation for SR-IOV */
>>>> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>>>> + }
>>>> +
>>>> + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>>>> + /* driver request VRAM reservation for SR-IOV */
>>>> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
>>>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>>>> + }
>>>> +
>>>> + *usage_bytes = 0;
>>>> + return 0;
>>>> +}
>>>> +
>>>> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
>>>> struct atom_context *ctx = adev->mode_info.atom_context;
>>>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>>>> vram_usagebyfirmware);
>>>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>>>> - uint32_t start_addr, size;
>>>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>>>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>>>> uint16_t data_offset;
>>>> + uint8_t frev, crev;
>>>> int usage_bytes = 0;
>>>>
>>>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>>>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>>>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>>>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>>>> -
>>>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>>>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>>>> -
>>>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>>>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>>>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>>>> - /* Firmware request VRAM reservation for SR-IOV */
>>>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>>>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>>>> - adev->mman.fw_vram_usage_size = size << 10;
>>>> - /* Use the default scratch size */
>>>> - usage_bytes = 0;
>>>> - } else {
>>>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>>>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>>>> + if (frev == 2 && crev == 1) {
>>>> + firmware_usage_v2_1 =
>>>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
>>>> + firmware_usage_v2_1,
>>>> + &usage_bytes);
>>>> + } else if (frev >= 2 && crev >= 2) {
>>>> + firmware_usage_v2_2 =
>>>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>>>> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
>>>> + firmware_usage_v2_2,
>>>> + &usage_bytes);
>>>> }
>>>> }
>>>> +
>>>> ctx->scratch_size_bytes = 0;
>>>> if (usage_bytes == 0)
>>>> usage_bytes = 20 * 1024; diff --git
>>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> index 585460ab8dfd..4a73cb314086 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>>>> NULL, &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/*
>>>> + * Driver Reservation functions
>>>> + */
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * free drv reserved vram if it has been reserved.
>>>> + */
>>>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
>>>> +*adev) {
>>>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL, NULL);
>>>> +}
>>>> +
>>>> /**
>>>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>>>> *
>>>> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>>>> &adev->mman.fw_vram_usage_va);
>>>> }
>>>>
>>>> +/**
>>>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation
>>>> +from driver
>>>> + *
>>>> + * @adev: amdgpu_device pointer
>>>> + *
>>>> + * create bo vram reservation from drv.
>>>> + */
>>>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
>>>> +*adev) {
>>>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>>>> +
>>>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>>>> +
>>>> + if (adev->mman.drv_vram_usage_size == 0 ||
>>>> + adev->mman.drv_vram_usage_size > vram_size)
>>>> + return 0;
>>>> +
>>>> + return amdgpu_bo_create_kernel_at(adev,
>>>> + adev->mman.drv_vram_usage_start_offset,
>>>> + adev->mman.drv_vram_usage_size,
>>>> + AMDGPU_GEM_DOMAIN_VRAM,
>>>> + &adev->mman.drv_vram_usage_reserved_bo,
>>>> + NULL); }
>>>> +
>>>> /*
>>>> * Memoy training reservation functions
>>>> */
>>>> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>>> return r;
>>>> }
>>>>
>>>> + /*
>>>> + *The reserved vram for driver must be pinned to the specified
>>>> + *place on the VRAM, so reserve it early.
>>>> + */
>>>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>>>> + if (r)
>>>> + return r;
>>>> +
>>>> /*
>>>> * only NAVI10 and onwards ASIC support for IP discovery.
>>>> * If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>>>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>>>> &adev->mman.sdma_access_ptr);
>>>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>>>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>>>
>>>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> index 9120ae80ef52..339838675b11 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>>>> @@ -92,6 +92,11 @@ struct amdgpu_mman {
>>>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>>>> void *fw_vram_usage_va;
>>>>
>>>> + /* driver VRAM reservation */
>>>> + u64 drv_vram_usage_start_offset;
>>>> + u64 drv_vram_usage_size;
>>>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>>>> +
>>>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>>>> struct amdgpu_bo *sdma_access_bo;
>>>> void *sdma_access_ptr;
>>>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> index ff855cb21d3f..c0f56ae653f0 100644
>>>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>>>> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
>>>>
>>>>
>>>> -/*
>>>> - ***************************************************************************
>>>> - Data Table vram_usagebyfirmware structure
>>>> -
>>>> ********************************************************************
>>>> **
>>>> *****
>>>> +/*
>>>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>>>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>>>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>>>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>>>> + Host driver would overwrite the table with the following
>>>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>>>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>>>> + } else {
>>>> + there is no VBIOS reservation region
>>>> + driver must allocate driver reservation region at top of FB.
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + }
>>>> + } else { //( NV1X and after)
>>>> + if( VBIOS/UEFI GOP is posted ) {
>>>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>>>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>>>> + }
>>>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>>>> + driver can allocate driver reservation region under firmware reservation,
>>>> + driver set used_by_driver_in_kb = driver reservation size
>>>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>>>> + same as Comment1
>>>> + } else {
>>>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>>>> + driver set used_by_driver_region0_in_kb = driver reservation size
>>>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>>>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>>>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>>>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>>>> + }
>>>> + }
>>>> */
>>>>
>>>> struct vram_usagebyfirmware_v2_1
>>>> {
>>>> - struct atom_common_table_header table_header;
>>>> - uint32_t start_address_in_kb;
>>>> - uint16_t used_by_firmware_in_kb;
>>>> - uint16_t used_by_driver_in_kb;
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t used_by_driver_in_kb;
>>>> };
>>>>
>>>> +struct vram_usagebyfirmware_v2_2 {
>>>> + struct atom_common_table_header table_header;
>>>> + uint32_t fw_region_start_address_in_kb;
>>>> + uint16_t used_by_firmware_in_kb;
>>>> + uint16_t reserved;
>>>> + uint32_t driver_region0_start_address_in_kb;
>>>> + uint32_t used_by_driver_region0_in_kb;
>>>> + uint32_t reserved32[7];
>>>> +};
>>>>
>>>> /*
>>>>
>>>> ********************************************************************
>>>> **
>>>> *****
>>>> --
>>>> 2.25.1
>>
^ permalink raw reply [flat|nested] 27+ messages in thread
* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-08 17:28 Bokun Zhang
0 siblings, 0 replies; 27+ messages in thread
From: Bokun Zhang @ 2022-11-08 17:28 UTC (permalink / raw)
To: amd-gfx; +Cc: Bokun Zhang, Tong Liu01
- Move TMR region from top of FB to 2MB for FFBM, so we need to
reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
Signed-off-by: Bokun Zhang <bokun.zhang@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 68 +++++++++--
4 files changed, 198 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..032dc2678d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, fw_size, drv_size;
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+ drv_size = le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb);
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ start_addr,
+ fw_size,
+ drv_size);
+
+ if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes = drv_size << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ fw_start_addr,
+ fw_size,
+ drv_start_addr,
+ drv_size);
+
+ if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 76a8ebfc9e71..06673a756231 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1579,6 +1579,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1605,6 +1621,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1772,6 +1813,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1897,6 +1946,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 6a70818039dd..7c38843f411e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -84,6 +84,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..68a86b14dff4 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,72 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer.
+ * driver should not write access that region.
+ * driver can allocate their own reservation region as long as it does not overlap firwmare's
+ * reservation region.
+ * if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ * in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ * if( VBIOS/UEFI GOP is posted ) {
+ * VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * Where: ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ * driver can allocate driver reservation region under firmware reservation,
+ * used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ * Comment1[hchan]:
+ * There is only one reservation at the beginning of the FB reserved by Host driver.
+ * Host driver would overwrite the table with the following
+ * used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ * set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ * } else {
+ * there is no VBIOS reservation region
+ * driver must allocate driver reservation region at top of FB.
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ * same as Comment1
+ * }
+ * } else { //( NV1X and after)
+ * if( VBIOS/UEFI GOP is posted ) {
+ * VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb;
+ * Where: ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ * }
+ * if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ * driver can allocate driver reservation region under firmware reservation,
+ * driver set used_by_driver_in_kb = driver reservation size
+ * driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ * same as Comment1
+ * } else {
+ * dirver can allocate it reservation any place as long as
+ * it does overlap pre-OS FW reservation area
+ * driver set used_by_driver_region0_in_kb = driver reservation size
+ * driver set driver_region0_start_address_in_kb = driver reservation region start address
+ * Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ * as the reservation for VF as it doesn’t exist. And Host driver should also
+ * update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ * }
+ * }
*/
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 2:46 Tong Liu01
2022-11-08 7:25 ` Liu01, Tong (Esther)
@ 2022-11-08 9:40 ` Christian König
1 sibling, 0 replies; 27+ messages in thread
From: Christian König @ 2022-11-08 9:40 UTC (permalink / raw)
To: Tong Liu01, amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tuikov Luben, Deucher Alexander, Evan Quan, Christian König,
Monk Liu, Hawking Zhang
Am 08.11.22 um 03:46 schrieb Tong Liu01:
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR
> region firstly to make sure TMR can be allocated at 2MB
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
> drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
> 4 files changed, 192 insertions(+), 31 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..239c621feb0a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
> }
> }
>
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
> + int *usage_bytes)
> +{
> + uint32_t start_addr, size;
> +
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
As I wrote before try to avoid calling le*_to_cpu multiple times.
In this case here get the start_addr and size and then print it, not the
other way around.
> +
> + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
This is still not looking like normal kernel coding style. What editor
are you using?
And please drop the explicit uint32_t conversion.
Regards,
Christian.
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = size << 10;
> + /* Use the default scratch size */
> + *usage_bytes = 0;
> + } else {
> + *usage_bytes =
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
> + }
> + return 0;
> +}
> +
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
> + int *usage_bytes)
> +{
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
> +
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
> +
> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
> + drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
> +
> + if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> + *usage_bytes = 0;
> + return 0;
> +}
> +
> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
> {
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> - usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
> + &usage_bytes);
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
> + &usage_bytes);
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..4a73cb314086 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
> +{
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> +{
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + NULL);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r)
> + return r;
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be
> @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..339838675b11 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,11 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..c0f56ae653f0 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1
> };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> - ***************************************************************************
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> {
> - struct atom_common_table_header table_header;
> - uint32_t start_address_in_kb;
> - uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + struct atom_common_table_header table_header;
> + uint32_t start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2 {
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
> ***************************************************************************
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-08 2:46 Tong Liu01
@ 2022-11-08 7:25 ` Liu01, Tong (Esther)
2022-11-08 9:40 ` Christian König
1 sibling, 0 replies; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-08 7:25 UTC (permalink / raw)
To: Liu01, Tong (Esther), amd-gfx, Koenig, Christian
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Chang, HaiJun, Tuikov, Luben, Sohail,
Rashid, Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang,
Hawking
[AMD Official Use Only - General]
Hi @Koenig, Christian,
Removed the code format style warning base on the checkpatch.pl script. Please help me review the new patch below, thanks!
Kind regards,
Esther
-----Original Message-----
From: Tong Liu01 <Tong.Liu01@amd.com>
Sent: 2022年11月8日星期二 上午10:47
To: amd-gfx@lists.freedesktop.org
Cc: Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
4 files changed, 192 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..239c621feb0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size =
+le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..4a73cb314086 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
+*adev) {
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
+driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be @@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..c0f56ae653f0 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1 };
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-08 2:46 Tong Liu01
2022-11-08 7:25 ` Liu01, Tong (Esther)
2022-11-08 9:40 ` Christian König
0 siblings, 2 replies; 27+ messages in thread
From: Tong Liu01 @ 2022-11-08 2:46 UTC (permalink / raw)
To: amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tong Liu01, Tuikov Luben, Deucher Alexander, Evan Quan,
Christian König, Monk Liu, Hawking Zhang
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR
region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 50 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 62 ++++++++--
4 files changed, 192 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..239c621feb0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..4a73cb314086 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r)
+ return r;
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1896,6 +1945,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..c0f56ae653f0 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,20 +705,66 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
{
- struct atom_common_table_header table_header;
- uint32_t start_address_in_kb;
- uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ struct atom_common_table_header table_header;
+ uint32_t start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2 {
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-07 3:32 Tong Liu01
2022-11-07 10:33 ` Liu01, Tong (Esther)
@ 2022-11-07 17:47 ` Christian König
1 sibling, 0 replies; 27+ messages in thread
From: Christian König @ 2022-11-07 17:47 UTC (permalink / raw)
To: Tong Liu01, amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tuikov Luben, Deucher Alexander, Evan Quan, Monk Liu,
Hawking Zhang
Am 07.11.22 um 04:32 schrieb Tong Liu01:
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region
> firstly to make sure TMR can be allocated at 2MB
At few coding style things below, but looks good to me from the
technically side.
Please use the checkpatch.pl script, it should point out a couple of issues.
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
> drivers/gpu/drm/amd/include/atomfirmware.h | 56 ++++++++-
> 4 files changed, 190 insertions(+), 28 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..239c621feb0a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
> }
> }
>
> +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
> + int *usage_bytes)
Could be that this is just my mail client, but of hand this doesn't
looks like normal kernel function style (but I might be wrong).
> +{
> + uint32_t start_addr, size;
> +
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
> +
> + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
Here again, this isn't normal kernel coding style.
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = size << 10;
> + /* Use the default scratch size */
> + *usage_bytes = 0;
> + } else {
> + *usage_bytes =
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
> + }
> + return 0;
> +}
> +
> +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
> + int *usage_bytes)
> +{
> + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
> +
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
While at it maybe convert this from le to cpu only once.
> +
> + fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
> + drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
In other words use those local variable for printing.
> +
> + if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> + *usage_bytes = 0;
> + return 0;
> +}
> +
> int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
> {
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> - usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_1(adev,
> + firmware_usage_v2_1,
> + &usage_bytes);
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + amdgpu_atomfirmware_allocate_fb_v2_2(adev,
> + firmware_usage_v2_2,
> + &usage_bytes);
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..b2779e68b734 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
> +{
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, NULL);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> +{
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + NULL);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1812,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r) {
> + return r;
> + }
No {} for single line statements after an if.
Christian.
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be
> @@ -1896,6 +1946,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..339838675b11 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,11 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..9f8761407099 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
> };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> - ***************************************************************************
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> @@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
> struct atom_common_table_header table_header;
> uint32_t start_address_in_kb;
> uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2{
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
> ***************************************************************************
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-07 3:32 Tong Liu01
@ 2022-11-07 10:33 ` Liu01, Tong (Esther)
2022-11-07 17:47 ` Christian König
1 sibling, 0 replies; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-07 10:33 UTC (permalink / raw)
To: amd-gfx, Koenig, Christian, Wang, Yang(Kevin)
Cc: Xiao, Jack, Xu, Feifei, Chen, Horace, Liu01, Tong (Esther),
Tuikov, Luben, Deucher, Alexander, Quan, Evan, Liu, Monk, Zhang,
Hawking
[AMD Official Use Only - General]
Hi @Koenig, Christian & @Wang, Yang(Kevin),
Refined patch based on your comments, please help review new patch below, thanks!
Kind regards,
Esther
-----Original Message-----
From: Tong Liu01 <Tong.Liu01@amd.com>
Sent: 2022年11月7日星期一 上午11:33
To: amd-gfx@lists.freedesktop.org
Cc: Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 56 ++++++++-
4 files changed, 190 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..239c621feb0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size =
+le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..b2779e68b734 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
+*adev) {
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
+driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be @@ -1896,6 +1946,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..9f8761407099 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1 };
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
@@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
struct atom_common_table_header table_header;
uint32_t start_address_in_kb;
uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2{
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-07 3:32 Tong Liu01
2022-11-07 10:33 ` Liu01, Tong (Esther)
2022-11-07 17:47 ` Christian König
0 siblings, 2 replies; 27+ messages in thread
From: Tong Liu01 @ 2022-11-07 3:32 UTC (permalink / raw)
To: amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tong Liu01, Tuikov Luben, Deucher Alexander, Evan Quan,
Christian König, Monk Liu, Hawking Zhang
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region
firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 +
drivers/gpu/drm/amd/include/atomfirmware.h | 56 ++++++++-
4 files changed, 190 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..239c621feb0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ *usage_bytes = 0;
+ } else {
+ *usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ *usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..b2779e68b734 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, NULL);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ NULL);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1812,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1896,6 +1946,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..339838675b11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,11 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..9f8761407099 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
@@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
struct atom_common_table_header table_header;
uint32_t start_address_in_kb;
uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2{
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-04 11:01 Tong Liu01
@ 2022-11-07 2:36 ` Liu01, Tong (Esther)
0 siblings, 0 replies; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-07 2:36 UTC (permalink / raw)
To: Liu01, Tong (Esther), amd-gfx, Koenig, Christian
Cc: Andrey Grodzovsky, Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Deucher, Alexander,
Quan, Evan, Liu, Monk, Zhang, Hawking
[AMD Official Use Only - General]
Hi @Koenig, Christian,
Can you help me review the new patch below? Thanks!
Kind regards,
Esther
-----Original Message-----
From: Tong Liu01 <Tong.Liu01@amd.com>
Sent: 2022年11月4日星期五 下午7:01
To: amd-gfx@lists.freedesktop.org
Cc: Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>; Liu01, Tong (Esther) <Tong.Liu01@amd.com>
Subject: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 +
drivers/gpu/drm/amd/include/atomfirmware.h | 56 ++++++++-
4 files changed, 192 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..65cf23818f4e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ usage_bytes = 0;
+ } else {
+ usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size =
+le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) {
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..099605380b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
+*adev) {
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, &adev->mman.drv_vram_usage_va); }
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from
+driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_va = NULL;
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ &adev->mman.drv_vram_usage_va);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be @@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..c60246f32f98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,12 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+ void *drv_vram_usage_va;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..9f8761407099 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1 };
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
@@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
struct atom_common_table_header table_header;
uint32_t start_address_in_kb;
uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2{
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-04 11:01 Tong Liu01
2022-11-07 2:36 ` Liu01, Tong (Esther)
0 siblings, 1 reply; 27+ messages in thread
From: Tong Liu01 @ 2022-11-04 11:01 UTC (permalink / raw)
To: amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tong Liu01, Tuikov Luben, Deucher Alexander, Evan Quan,
Christian König, Monk Liu, Hawking Zhang
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region
firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 106 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 +++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 +
drivers/gpu/drm/amd/include/atomfirmware.h | 56 ++++++++-
4 files changed, 192 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..65cf23818f4e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -101,39 +101,99 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
}
}
+static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1,
+ int *usage_bytes)
+{
+ uint32_t start_addr, size;
+
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ usage_bytes = 0;
+ } else {
+ usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ return 0;
+}
+
+static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev,
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2,
+ int *usage_bytes)
+{
+ uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size;
+
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ fw_start_addr = le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ drv_start_addr = le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ if ((uint32_t)(fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (fw_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)(drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = (drv_start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
+ usage_bytes = 0;
+ return 0;
+}
+
int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
{
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
- usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_1(adev,
+ firmware_usage_v2_1,
+ &usage_bytes);
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ amdgpu_atomfirmware_allocate_fb_v2_2(adev,
+ firmware_usage_v2_2,
+ &usage_bytes);
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..099605380b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, &adev->mman.drv_vram_usage_va);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_va = NULL;
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ &adev->mman.drv_vram_usage_va);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..c60246f32f98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,12 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+ void *drv_vram_usage_va;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..9f8761407099 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
@@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
struct atom_common_table_header table_header;
uint32_t start_address_in_kb;
uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2{
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-04 9:34 ` Liu01, Tong (Esther)
@ 2022-11-04 9:39 ` Christian König
0 siblings, 0 replies; 27+ messages in thread
From: Christian König @ 2022-11-04 9:39 UTC (permalink / raw)
To: Liu01, Tong (Esther), amd-gfx
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Deucher, Alexander,
Quan, Evan, Liu, Monk, Zhang, Hawking
Hi Esther,
ok, that works as well.
I would just move the handling of version of the firmware table into
separate functions. That should make it easier to understand what's
going on here.
Thanks,
Christian.
Am 04.11.22 um 10:34 schrieb Liu01, Tong (Esther):
> [AMD Official Use Only - General]
>
> Hi Christian,
>
> Based on VBIOS's comment " driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region". Our understanding is that they are not mutual exclusive. So we create extra parameters to record drv reservation request.
>
> Kind regards,
> Esther
>
> -----Original Message-----
> From: Koenig, Christian <Christian.Koenig@amd.com>
> Sent: 2022年11月4日星期五 下午5:09
> To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
>
> Am 04.11.22 um 09:52 schrieb Tong Liu01:
>> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
>> TMR region firstly to make sure TMR can be allocated at 2MB
> If I understand it correctly the two methods are mutual exclusive. So I think you can just extend the existing function in the amdgpu TTM code instead of adding a new one.
>
>> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
>> ---
>> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 84 ++++++++++++++-----
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 ++++++++++++
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++
>> drivers/gpu/drm/amd/include/atomfirmware.h | 56 +++++++++++--
>> 4 files changed, 171 insertions(+), 27 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> index b81b77a9efa6..f577b1d151d9 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
>> @@ -106,34 +106,74 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
>> struct atom_context *ctx = adev->mode_info.atom_context;
>> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
>> vram_usagebyfirmware);
>> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
>> - uint32_t start_addr, size;
>> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
>> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
>> + uint32_t start_addr, size, fw_start_addr, fw_size, drv_addr, drv_size;
>> uint16_t data_offset;
>> + uint8_t frev, crev;
>> int usage_bytes = 0;
>>
>> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
>> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
>> - le32_to_cpu(firmware_usage->start_address_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
>> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
>> -
>> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
>> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
>> -
>> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> - /* Firmware request VRAM reservation for SR-IOV */
>> - adev->mman.fw_vram_usage_start_offset = (start_addr &
>> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> - adev->mman.fw_vram_usage_size = size << 10;
>> - /* Use the default scratch size */
>> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
>> + if (frev == 2 && crev == 1) {
>> + firmware_usage_v2_1 =
>> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
>> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
>> + le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
>> + le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
>> +
>> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
>> + size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
>> +
>> + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
>> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
>> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = (start_addr &
>> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
>> + adev->mman.fw_vram_usage_size = size << 10;
>> + /* Use the default scratch size */
>> + usage_bytes = 0;
>> + } else {
>> + usage_bytes =
>> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
>> + }
> That should probably be put into separate functions.
>
> Regards,
> Christian.
>
>> + } else if (frev >= 2 && crev >= 2) {
>> + firmware_usage_v2_2 =
>> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
>> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
>> + le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
>> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
>> + le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
>> + le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
>> +
>> + if ((uint32_t)((le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
>> + & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
>> + fw_start_addr = (le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
>> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
>> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
>> +
>> + /* Firmware request VRAM reservation for SR-IOV */
>> + adev->mman.fw_vram_usage_start_offset = fw_start_addr << 10;
>> + adev->mman.fw_vram_usage_size = fw_size << 10;
>> + }
>> +
>> + if ((uint32_t)((ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)
>> + & (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))) == 0) {
>> + drv_addr = (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))
>> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
>> + drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
>> +
>> + /* driver request VRAM reservation for SR-IOV */
>> + adev->mman.drv_vram_usage_start_offset = drv_addr << 10;
>> + adev->mman.drv_vram_usage_size = drv_size << 10;
>> + }
>> +
>> usage_bytes = 0;
>> - } else {
>> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
>> +
>> }
>> }
>> +
>> ctx->scratch_size_bytes = 0;
>> if (usage_bytes == 0)
>> usage_bytes = 20 * 1024;
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> index 585460ab8dfd..099605380b06 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
>> NULL, &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/*
>> + * Driver Reservation functions
>> + */
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * free drv reserved vram if it has been reserved.
>> + */
>> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
>> +{
>> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
>> + NULL, &adev->mman.drv_vram_usage_va);
>> +}
>> +
>> /**
>> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
>> *
>> @@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>> &adev->mman.fw_vram_usage_va);
>> }
>>
>> +/**
>> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
>> + *
>> + * @adev: amdgpu_device pointer
>> + *
>> + * create bo vram reservation from drv.
>> + */
>> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
>> +{
>> + uint64_t vram_size = adev->gmc.visible_vram_size;
>> +
>> + adev->mman.drv_vram_usage_va = NULL;
>> + adev->mman.drv_vram_usage_reserved_bo = NULL;
>> +
>> + if (adev->mman.drv_vram_usage_size == 0 ||
>> + adev->mman.drv_vram_usage_size > vram_size)
>> + return 0;
>> +
>> + return amdgpu_bo_create_kernel_at(adev,
>> + adev->mman.drv_vram_usage_start_offset,
>> + adev->mman.drv_vram_usage_size,
>> + AMDGPU_GEM_DOMAIN_VRAM,
>> + &adev->mman.drv_vram_usage_reserved_bo,
>> + &adev->mman.drv_vram_usage_va);
>> +}
>> +
>> /*
>> * Memoy training reservation functions
>> */
>> @@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>> return r;
>> }
>>
>> + /*
>> + *The reserved vram for driver must be pinned to the specified
>> + *place on the VRAM, so reserve it early.
>> + */
>> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
>> + if (r) {
>> + return r;
>> + }
>> +
>> /*
>> * only NAVI10 and onwards ASIC support for IP discovery.
>> * If IP discovery enabled, a block of memory should be
>> @@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
>> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
>> &adev->mman.sdma_access_ptr);
>> amdgpu_ttm_fw_reserve_vram_fini(adev);
>> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>>
>> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> index 9120ae80ef52..c60246f32f98 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
>> @@ -92,6 +92,12 @@ struct amdgpu_mman {
>> struct amdgpu_bo *fw_vram_usage_reserved_bo;
>> void *fw_vram_usage_va;
>>
>> + /* driver VRAM reservation */
>> + u64 drv_vram_usage_start_offset;
>> + u64 drv_vram_usage_size;
>> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
>> + void *drv_vram_usage_va;
>> +
>> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
>> struct amdgpu_bo *sdma_access_bo;
>> void *sdma_access_ptr;
>> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
>> index ff855cb21d3f..9f8761407099 100644
>> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
>> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
>> @@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
>> };
>>
>>
>> -/*
>> - ***************************************************************************
>> - Data Table vram_usagebyfirmware structure
>> - ***************************************************************************
>> +/*
>> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
>> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
>> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
>> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
>> + driver can allocate driver reservation region under firmware reservation,
>> + used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
>> + Host driver would overwrite the table with the following
>> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
>> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
>> + } else {
>> + there is no VBIOS reservation region
>> + driver must allocate driver reservation region at top of FB.
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + }
>> + } else { //( NV1X and after)
>> + if( VBIOS/UEFI GOP is posted ) {
>> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
>> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
>> + }
>> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
>> + driver can allocate driver reservation region under firmware reservation,
>> + driver set used_by_driver_in_kb = driver reservation size
>> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
>> + same as Comment1
>> + } else {
>> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
>> + driver set used_by_driver_region0_in_kb = driver reservation size
>> + driver set driver_region0_start_address_in_kb = driver reservation region start address
>> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
>> + as the reservation for VF as it doesn’t exist. And Host driver should also
>> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
>> + }
>> + }
>> */
>>
>> struct vram_usagebyfirmware_v2_1
>> @@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
>> struct atom_common_table_header table_header;
>> uint32_t start_address_in_kb;
>> uint16_t used_by_firmware_in_kb;
>> - uint16_t used_by_driver_in_kb;
>> + uint16_t used_by_driver_in_kb;
>> };
>>
>> +struct vram_usagebyfirmware_v2_2{
>> + struct atom_common_table_header table_header;
>> + uint32_t fw_region_start_address_in_kb;
>> + uint16_t used_by_firmware_in_kb;
>> + uint16_t reserved;
>> + uint32_t driver_region0_start_address_in_kb;
>> + uint32_t used_by_driver_region0_in_kb;
>> + uint32_t reserved32[7];
>> +};
>>
>> /*
>> ***************************************************************************
^ permalink raw reply [flat|nested] 27+ messages in thread
* RE: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-04 9:09 ` Christian König
@ 2022-11-04 9:34 ` Liu01, Tong (Esther)
2022-11-04 9:39 ` Christian König
0 siblings, 1 reply; 27+ messages in thread
From: Liu01, Tong (Esther) @ 2022-11-04 9:34 UTC (permalink / raw)
To: Koenig, Christian, amd-gfx
Cc: Xiao, Jack, Wang, Yang(Kevin),
Xu, Feifei, Chen, Horace, Tuikov, Luben, Deucher, Alexander,
Quan, Evan, Liu, Monk, Zhang, Hawking
[AMD Official Use Only - General]
Hi Christian,
Based on VBIOS's comment " driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region". Our understanding is that they are not mutual exclusive. So we create extra parameters to record drv reservation request.
Kind regards,
Esther
-----Original Message-----
From: Koenig, Christian <Christian.Koenig@amd.com>
Sent: 2022年11月4日星期五 下午5:09
To: Liu01, Tong (Esther) <Tong.Liu01@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Andrey Grodzovsky <Andrey.Grodzovsky@amd.com>; Quan, Evan <Evan.Quan@amd.com>; Chen, Horace <Horace.Chen@amd.com>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Xiao, Jack <Jack.Xiao@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Wang, Yang(Kevin) <KevinYang.Wang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
Am 04.11.22 um 09:52 schrieb Tong Liu01:
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve
> TMR region firstly to make sure TMR can be allocated at 2MB
If I understand it correctly the two methods are mutual exclusive. So I think you can just extend the existing function in the amdgpu TTM code instead of adding a new one.
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 84 ++++++++++++++-----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 ++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++
> drivers/gpu/drm/amd/include/atomfirmware.h | 56 +++++++++++--
> 4 files changed, 171 insertions(+), 27 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..f577b1d151d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -106,34 +106,74 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> + uint32_t start_addr, size, fw_start_addr, fw_size, drv_addr, drv_size;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
> +
> + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = size << 10;
> + /* Use the default scratch size */
> + usage_bytes = 0;
> + } else {
> + usage_bytes =
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
> + }
That should probably be put into separate functions.
Regards,
Christian.
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
> +
> + if ((uint32_t)((le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
> + & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + fw_start_addr = (le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = fw_start_addr << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((uint32_t)((ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)
> + & (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))) == 0) {
> + drv_addr = (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))
> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
> + drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
> +
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = drv_addr << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> +
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..099605380b06 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
> +{
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, &adev->mman.drv_vram_usage_va);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> +{
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_va = NULL;
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + &adev->mman.drv_vram_usage_va);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r) {
> + return r;
> + }
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be
> @@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..c60246f32f98 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,12 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> + void *drv_vram_usage_va;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..9f8761407099 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
> };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> - ***************************************************************************
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> @@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
> struct atom_common_table_header table_header;
> uint32_t start_address_in_kb;
> uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2{
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
> ***************************************************************************
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
2022-11-04 8:52 Tong Liu01
@ 2022-11-04 9:09 ` Christian König
2022-11-04 9:34 ` Liu01, Tong (Esther)
0 siblings, 1 reply; 27+ messages in thread
From: Christian König @ 2022-11-04 9:09 UTC (permalink / raw)
To: Tong Liu01, amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tuikov Luben, Deucher Alexander, Evan Quan, Monk Liu,
Hawking Zhang
Am 04.11.22 um 09:52 schrieb Tong Liu01:
> Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region
> firstly to make sure TMR can be allocated at 2MB
If I understand it correctly the two methods are mutual exclusive. So I
think you can just extend the existing function in the amdgpu TTM code
instead of adding a new one.
>
> Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
> ---
> .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 84 ++++++++++++++-----
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 ++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++
> drivers/gpu/drm/amd/include/atomfirmware.h | 56 +++++++++++--
> 4 files changed, 171 insertions(+), 27 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> index b81b77a9efa6..f577b1d151d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
> @@ -106,34 +106,74 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
> struct atom_context *ctx = adev->mode_info.atom_context;
> int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
> vram_usagebyfirmware);
> - struct vram_usagebyfirmware_v2_1 *firmware_usage;
> - uint32_t start_addr, size;
> + struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
> + struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
> + uint32_t start_addr, size, fw_start_addr, fw_size, drv_addr, drv_size;
> uint16_t data_offset;
> + uint8_t frev, crev;
> int usage_bytes = 0;
>
> - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
> - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
> - le32_to_cpu(firmware_usage->start_address_in_kb),
> - le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
> - le16_to_cpu(firmware_usage->used_by_driver_in_kb));
> -
> - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
> - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
> -
> - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> - /* Firmware request VRAM reservation for SR-IOV */
> - adev->mman.fw_vram_usage_start_offset = (start_addr &
> - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> - adev->mman.fw_vram_usage_size = size << 10;
> - /* Use the default scratch size */
> + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
> + if (frev == 2 && crev == 1) {
> + firmware_usage_v2_1 =
> + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
> + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
> + le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
> +
> + start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
> + size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
> +
> + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
> + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
> + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = (start_addr &
> + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
> + adev->mman.fw_vram_usage_size = size << 10;
> + /* Use the default scratch size */
> + usage_bytes = 0;
> + } else {
> + usage_bytes =
> + le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
> + }
That should probably be put into separate functions.
Regards,
Christian.
> + } else if (frev >= 2 && crev >= 2) {
> + firmware_usage_v2_2 =
> + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
> + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
> + le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
> + le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
> + le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
> +
> + if ((uint32_t)((le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
> + & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
> + fw_start_addr = (le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
> + fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
> +
> + /* Firmware request VRAM reservation for SR-IOV */
> + adev->mman.fw_vram_usage_start_offset = fw_start_addr << 10;
> + adev->mman.fw_vram_usage_size = fw_size << 10;
> + }
> +
> + if ((uint32_t)((ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)
> + & (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))) == 0) {
> + drv_addr = (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))
> + & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
> + drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
> +
> + /* driver request VRAM reservation for SR-IOV */
> + adev->mman.drv_vram_usage_start_offset = drv_addr << 10;
> + adev->mman.drv_vram_usage_size = drv_size << 10;
> + }
> +
> usage_bytes = 0;
> - } else {
> - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
> +
> }
> }
> +
> ctx->scratch_size_bytes = 0;
> if (usage_bytes == 0)
> usage_bytes = 20 * 1024;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 585460ab8dfd..099605380b06 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
> NULL, &adev->mman.fw_vram_usage_va);
> }
>
> +/*
> + * Driver Reservation functions
> + */
> +/**
> + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * free drv reserved vram if it has been reserved.
> + */
> +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
> +{
> + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
> + NULL, &adev->mman.drv_vram_usage_va);
> +}
> +
> /**
> * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
> *
> @@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
> &adev->mman.fw_vram_usage_va);
> }
>
> +/**
> + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
> + *
> + * @adev: amdgpu_device pointer
> + *
> + * create bo vram reservation from drv.
> + */
> +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
> +{
> + uint64_t vram_size = adev->gmc.visible_vram_size;
> +
> + adev->mman.drv_vram_usage_va = NULL;
> + adev->mman.drv_vram_usage_reserved_bo = NULL;
> +
> + if (adev->mman.drv_vram_usage_size == 0 ||
> + adev->mman.drv_vram_usage_size > vram_size)
> + return 0;
> +
> + return amdgpu_bo_create_kernel_at(adev,
> + adev->mman.drv_vram_usage_start_offset,
> + adev->mman.drv_vram_usage_size,
> + AMDGPU_GEM_DOMAIN_VRAM,
> + &adev->mman.drv_vram_usage_reserved_bo,
> + &adev->mman.drv_vram_usage_va);
> +}
> +
> /*
> * Memoy training reservation functions
> */
> @@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
> return r;
> }
>
> + /*
> + *The reserved vram for driver must be pinned to the specified
> + *place on the VRAM, so reserve it early.
> + */
> + r = amdgpu_ttm_drv_reserve_vram_init(adev);
> + if (r) {
> + return r;
> + }
> +
> /*
> * only NAVI10 and onwards ASIC support for IP discovery.
> * If IP discovery enabled, a block of memory should be
> @@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
> amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
> &adev->mman.sdma_access_ptr);
> amdgpu_ttm_fw_reserve_vram_fini(adev);
> + amdgpu_ttm_drv_reserve_vram_fini(adev);
>
> if (drm_dev_enter(adev_to_drm(adev), &idx)) {
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> index 9120ae80ef52..c60246f32f98 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
> @@ -92,6 +92,12 @@ struct amdgpu_mman {
> struct amdgpu_bo *fw_vram_usage_reserved_bo;
> void *fw_vram_usage_va;
>
> + /* driver VRAM reservation */
> + u64 drv_vram_usage_start_offset;
> + u64 drv_vram_usage_size;
> + struct amdgpu_bo *drv_vram_usage_reserved_bo;
> + void *drv_vram_usage_va;
> +
> /* PAGE_SIZE'd BO for process memory r/w over SDMA. */
> struct amdgpu_bo *sdma_access_bo;
> void *sdma_access_ptr;
> diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
> index ff855cb21d3f..9f8761407099 100644
> --- a/drivers/gpu/drm/amd/include/atomfirmware.h
> +++ b/drivers/gpu/drm/amd/include/atomfirmware.h
> @@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
> };
>
>
> -/*
> - ***************************************************************************
> - Data Table vram_usagebyfirmware structure
> - ***************************************************************************
> +/*
> + VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
> + driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
> + if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
> + in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
> + driver can allocate driver reservation region under firmware reservation,
> + used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
> + Host driver would overwrite the table with the following
> + used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
> + set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
> + } else {
> + there is no VBIOS reservation region
> + driver must allocate driver reservation region at top of FB.
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + }
> + } else { //( NV1X and after)
> + if( VBIOS/UEFI GOP is posted ) {
> + VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
> + update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
> + }
> + if( vram_usagebyfirmwareTable version <= 2.1 ) {
> + driver can allocate driver reservation region under firmware reservation,
> + driver set used_by_driver_in_kb = driver reservation size
> + driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
> + same as Comment1
> + } else {
> + dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
> + driver set used_by_driver_region0_in_kb = driver reservation size
> + driver set driver_region0_start_address_in_kb = driver reservation region start address
> + Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
> + as the reservation for VF as it doesn’t exist. And Host driver should also
> + update atom_firmware_Info table to remove the same VBIOS reservation as well.
> + }
> + }
> */
>
> struct vram_usagebyfirmware_v2_1
> @@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
> struct atom_common_table_header table_header;
> uint32_t start_address_in_kb;
> uint16_t used_by_firmware_in_kb;
> - uint16_t used_by_driver_in_kb;
> + uint16_t used_by_driver_in_kb;
> };
>
> +struct vram_usagebyfirmware_v2_2{
> + struct atom_common_table_header table_header;
> + uint32_t fw_region_start_address_in_kb;
> + uint16_t used_by_firmware_in_kb;
> + uint16_t reserved;
> + uint32_t driver_region0_start_address_in_kb;
> + uint32_t used_by_driver_region0_in_kb;
> + uint32_t reserved32[7];
> +};
>
> /*
> ***************************************************************************
^ permalink raw reply [flat|nested] 27+ messages in thread
* [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2
@ 2022-11-04 8:52 Tong Liu01
2022-11-04 9:09 ` Christian König
0 siblings, 1 reply; 27+ messages in thread
From: Tong Liu01 @ 2022-11-04 8:52 UTC (permalink / raw)
To: amd-gfx
Cc: Andrey Grodzovsky, Jack Xiao, Feifei Xu, horace.chen, Kevin Wang,
Tong Liu01, Tuikov Luben, Deucher Alexander, Evan Quan,
Christian König, Monk Liu, Hawking Zhang
Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region
firstly to make sure TMR can be allocated at 2MB
Signed-off-by: Tong Liu01 <Tong.Liu01@amd.com>
---
.../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 84 ++++++++++++++-----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 52 ++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++
drivers/gpu/drm/amd/include/atomfirmware.h | 56 +++++++++++--
4 files changed, 171 insertions(+), 27 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b81b77a9efa6..f577b1d151d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -106,34 +106,74 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
struct atom_context *ctx = adev->mode_info.atom_context;
int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
vram_usagebyfirmware);
- struct vram_usagebyfirmware_v2_1 *firmware_usage;
- uint32_t start_addr, size;
+ struct vram_usagebyfirmware_v2_1 *firmware_usage_v2_1;
+ struct vram_usagebyfirmware_v2_2 *firmware_usage_v2_2;
+ uint32_t start_addr, size, fw_start_addr, fw_size, drv_addr, drv_size;
uint16_t data_offset;
+ uint8_t frev, crev;
int usage_bytes = 0;
- if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
- firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
- DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n",
- le32_to_cpu(firmware_usage->start_address_in_kb),
- le16_to_cpu(firmware_usage->used_by_firmware_in_kb),
- le16_to_cpu(firmware_usage->used_by_driver_in_kb));
-
- start_addr = le32_to_cpu(firmware_usage->start_address_in_kb);
- size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb);
-
- if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
- (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
- ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
- /* Firmware request VRAM reservation for SR-IOV */
- adev->mman.fw_vram_usage_start_offset = (start_addr &
- (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
- adev->mman.fw_vram_usage_size = size << 10;
- /* Use the default scratch size */
+ if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) {
+ if (frev == 2 && crev == 1) {
+ firmware_usage_v2_1 =
+ (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset);
+ DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n",
+ le32_to_cpu(firmware_usage_v2_1->start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb),
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb));
+
+ start_addr = le32_to_cpu(firmware_usage_v2_1->start_address_in_kb);
+ size = le16_to_cpu(firmware_usage_v2_1->used_by_firmware_in_kb);
+
+ if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) ==
+ (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION <<
+ ATOM_VRAM_OPERATION_FLAGS_SHIFT)) {
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = (start_addr &
+ (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10;
+ adev->mman.fw_vram_usage_size = size << 10;
+ /* Use the default scratch size */
+ usage_bytes = 0;
+ } else {
+ usage_bytes =
+ le16_to_cpu(firmware_usage_v2_1->used_by_driver_in_kb) << 10;
+ }
+ } else if (frev >= 2 && crev >= 2) {
+ firmware_usage_v2_2 =
+ (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset);
+ DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n",
+ le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb),
+ le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb),
+ le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb));
+
+ if ((uint32_t)((le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
+ & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) {
+ fw_start_addr = (le32_to_cpu(firmware_usage_v2_2->fw_region_start_address_in_kb))
+ & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
+ fw_size = le16_to_cpu(firmware_usage_v2_2->used_by_firmware_in_kb);
+
+ /* Firmware request VRAM reservation for SR-IOV */
+ adev->mman.fw_vram_usage_start_offset = fw_start_addr << 10;
+ adev->mman.fw_vram_usage_size = fw_size << 10;
+ }
+
+ if ((uint32_t)((ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)
+ & (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))) == 0) {
+ drv_addr = (le32_to_cpu(firmware_usage_v2_2->driver_region0_start_address_in_kb))
+ & (~ATOM_VRAM_OPERATION_FLAGS_MASK);
+ drv_size = le32_to_cpu(firmware_usage_v2_2->used_by_driver_region0_in_kb);
+
+ /* driver request VRAM reservation for SR-IOV */
+ adev->mman.drv_vram_usage_start_offset = drv_addr << 10;
+ adev->mman.drv_vram_usage_size = drv_size << 10;
+ }
+
usage_bytes = 0;
- } else {
- usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10;
+
}
}
+
ctx->scratch_size_bytes = 0;
if (usage_bytes == 0)
usage_bytes = 20 * 1024;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 585460ab8dfd..099605380b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1578,6 +1578,22 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
NULL, &adev->mman.fw_vram_usage_va);
}
+/*
+ * Driver Reservation functions
+ */
+/**
+ * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free drv reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
+ NULL, &adev->mman.drv_vram_usage_va);
+}
+
/**
* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
*
@@ -1604,6 +1620,32 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
&adev->mman.fw_vram_usage_va);
}
+/**
+ * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from drv.
+ */
+static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
+{
+ uint64_t vram_size = adev->gmc.visible_vram_size;
+
+ adev->mman.drv_vram_usage_va = NULL;
+ adev->mman.drv_vram_usage_reserved_bo = NULL;
+
+ if (adev->mman.drv_vram_usage_size == 0 ||
+ adev->mman.drv_vram_usage_size > vram_size)
+ return 0;
+
+ return amdgpu_bo_create_kernel_at(adev,
+ adev->mman.drv_vram_usage_start_offset,
+ adev->mman.drv_vram_usage_size,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ &adev->mman.drv_vram_usage_reserved_bo,
+ &adev->mman.drv_vram_usage_va);
+}
+
/*
* Memoy training reservation functions
*/
@@ -1771,6 +1813,15 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
return r;
}
+ /*
+ *The reserved vram for driver must be pinned to the specified
+ *place on the VRAM, so reserve it early.
+ */
+ r = amdgpu_ttm_drv_reserve_vram_init(adev);
+ if (r) {
+ return r;
+ }
+
/*
* only NAVI10 and onwards ASIC support for IP discovery.
* If IP discovery enabled, a block of memory should be
@@ -1896,6 +1947,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
&adev->mman.sdma_access_ptr);
amdgpu_ttm_fw_reserve_vram_fini(adev);
+ amdgpu_ttm_drv_reserve_vram_fini(adev);
if (drm_dev_enter(adev_to_drm(adev), &idx)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9120ae80ef52..c60246f32f98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -92,6 +92,12 @@ struct amdgpu_mman {
struct amdgpu_bo *fw_vram_usage_reserved_bo;
void *fw_vram_usage_va;
+ /* driver VRAM reservation */
+ u64 drv_vram_usage_start_offset;
+ u64 drv_vram_usage_size;
+ struct amdgpu_bo *drv_vram_usage_reserved_bo;
+ void *drv_vram_usage_va;
+
/* PAGE_SIZE'd BO for process memory r/w over SDMA. */
struct amdgpu_bo *sdma_access_bo;
void *sdma_access_ptr;
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index ff855cb21d3f..9f8761407099 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -705,10 +705,47 @@ struct atom_gpio_pin_lut_v2_1
};
-/*
- ***************************************************************************
- Data Table vram_usagebyfirmware structure
- ***************************************************************************
+/*
+ VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write access that region.
+ driver can allocate their own reservation region as long as it does not overlap firwmare's reservation region.
+ if( atom data table firmwareInfoTable version < 3.3) { //( pre-NV1X )
+ in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10)
+ driver can allocate driver reservation region under firmware reservation,
+ used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by Host driver.
+ Host driver would overwrite the table with the following
+ used_by_firmware_in_kb = total reserved size for pf-vf info exchange and
+ set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0
+ } else {
+ there is no VBIOS reservation region
+ driver must allocate driver reservation region at top of FB.
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ }
+ } else { //( NV1X and after)
+ if( VBIOS/UEFI GOP is posted ) {
+ VBIOS/UEFIGOP update used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb;
+ update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10 )
+ }
+ if( vram_usagebyfirmwareTable version <= 2.1 ) {
+ driver can allocate driver reservation region under firmware reservation,
+ driver set used_by_driver_in_kb = driver reservation size
+ driver reservation start address = (start_address_in_kb - used_by_driver_in_kb)
+ same as Comment1
+ } else {
+ dirver can allocate it reservation any place as long as it does overlap pre-OS FW reservation area
+ driver set used_by_driver_region0_in_kb = driver reservation size
+ driver set driver_region0_start_address_in_kb = driver reservation region start address
+ Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to zero
+ as the reservation for VF as it doesn’t exist. And Host driver should also
+ update atom_firmware_Info table to remove the same VBIOS reservation as well.
+ }
+ }
*/
struct vram_usagebyfirmware_v2_1
@@ -716,9 +753,18 @@ struct vram_usagebyfirmware_v2_1
struct atom_common_table_header table_header;
uint32_t start_address_in_kb;
uint16_t used_by_firmware_in_kb;
- uint16_t used_by_driver_in_kb;
+ uint16_t used_by_driver_in_kb;
};
+struct vram_usagebyfirmware_v2_2{
+ struct atom_common_table_header table_header;
+ uint32_t fw_region_start_address_in_kb;
+ uint16_t used_by_firmware_in_kb;
+ uint16_t reserved;
+ uint32_t driver_region0_start_address_in_kb;
+ uint32_t used_by_driver_region0_in_kb;
+ uint32_t reserved32[7];
+};
/*
***************************************************************************
--
2.25.1
^ permalink raw reply related [flat|nested] 27+ messages in thread
end of thread, other threads:[~2022-11-10 15:11 UTC | newest]
Thread overview: 27+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-11-08 10:32 [PATCH] drm/amdgpu: add vram reservation logic based on vram_usagebyfirmware_v2_2 Tong Liu01
2022-11-08 10:40 ` Liu01, Tong (Esther)
2022-11-08 12:53 ` Christian König
2022-11-08 13:49 ` Chang, HaiJun
2022-11-08 14:04 ` Christian König
2022-11-08 16:33 ` Luben Tuikov
2022-11-10 5:25 ` Liu01, Tong (Esther)
2022-11-10 9:49 ` Christian König
2022-11-10 15:10 ` Luben Tuikov
2022-11-10 10:14 ` Liu01, Tong (Esther)
2022-11-10 10:18 ` Christian König
2022-11-10 10:47 ` Liu01, Tong (Esther)
2022-11-10 10:49 ` Christian König
2022-11-10 12:17 ` Lazar, Lijo
-- strict thread matches above, loose matches on Subject: below --
2022-11-08 17:28 Bokun Zhang
2022-11-08 2:46 Tong Liu01
2022-11-08 7:25 ` Liu01, Tong (Esther)
2022-11-08 9:40 ` Christian König
2022-11-07 3:32 Tong Liu01
2022-11-07 10:33 ` Liu01, Tong (Esther)
2022-11-07 17:47 ` Christian König
2022-11-04 11:01 Tong Liu01
2022-11-07 2:36 ` Liu01, Tong (Esther)
2022-11-04 8:52 Tong Liu01
2022-11-04 9:09 ` Christian König
2022-11-04 9:34 ` Liu01, Tong (Esther)
2022-11-04 9:39 ` Christian König
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.