* [PATCH 02/11] habanalabs: rename mmu.c to mmu_v1.c
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 03/11] habanalabs: refactor MMU as device-oriented Oded Gabbay
` (8 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Moti Haimovski
From: Moti Haimovski <mhaimovski@habana.ai>
In the future we will have MMU v2 code, so we need to prepare the
driver for it. The first step is to rename the current MMU file to
mmu_v1.c.
Signed-off-by: Moti Haimovski <mhaimovski@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/common/Makefile | 4 ++--
drivers/misc/habanalabs/common/{mmu.c => mmu_v1.c} | 0
2 files changed, 2 insertions(+), 2 deletions(-)
rename drivers/misc/habanalabs/common/{mmu.c => mmu_v1.c} (100%)
diff --git a/drivers/misc/habanalabs/common/Makefile b/drivers/misc/habanalabs/common/Makefile
index b984bfa4face..e8ed12810517 100644
--- a/drivers/misc/habanalabs/common/Makefile
+++ b/drivers/misc/habanalabs/common/Makefile
@@ -3,5 +3,5 @@ HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \
common/asid.o common/habanalabs_ioctl.o \
common/command_buffer.o common/hw_queue.o common/irq.o \
common/sysfs.o common/hwmon.o common/memory.o \
- common/command_submission.o common/mmu.o common/firmware_if.o \
- common/pci.o
+ common/command_submission.o common/mmu_v1.o \
+ common/firmware_if.o common/pci.o
diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu_v1.c
similarity index 100%
rename from drivers/misc/habanalabs/common/mmu.c
rename to drivers/misc/habanalabs/common/mmu_v1.c
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 03/11] habanalabs: refactor MMU as device-oriented
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
2020-09-10 18:15 ` [PATCH 02/11] habanalabs: rename mmu.c to mmu_v1.c Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 04/11] habanalabs: add num_hops to hl_mmu_properties Oded Gabbay
` (7 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Moti Haimovski
From: Moti Haimovski <mhaimovski@habana.ai>
As preparation to MMU v2, rework MMU to be device oriented
instantiated according to the device in hand.
Signed-off-by: Moti Haimovski <mhaimovski@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/common/Makefile | 2 +-
drivers/misc/habanalabs/common/device.c | 6 +
drivers/misc/habanalabs/common/habanalabs.h | 49 +++-
drivers/misc/habanalabs/common/mmu.c | 309 ++++++++++++++++++++
drivers/misc/habanalabs/common/mmu_v1.c | 260 +++-------------
5 files changed, 404 insertions(+), 222 deletions(-)
create mode 100644 drivers/misc/habanalabs/common/mmu.c
diff --git a/drivers/misc/habanalabs/common/Makefile b/drivers/misc/habanalabs/common/Makefile
index e8ed12810517..eccd8c7dc62d 100644
--- a/drivers/misc/habanalabs/common/Makefile
+++ b/drivers/misc/habanalabs/common/Makefile
@@ -3,5 +3,5 @@ HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \
common/asid.o common/habanalabs_ioctl.o \
common/command_buffer.o common/hw_queue.o common/irq.o \
common/sysfs.o common/hwmon.o common/memory.o \
- common/command_submission.o common/mmu_v1.o \
+ common/command_submission.o common/mmu.o common/mmu_v1.o \
common/firmware_if.o common/pci.o
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
index aa7fa9e94651..196e35d71118 100644
--- a/drivers/misc/habanalabs/common/device.c
+++ b/drivers/misc/habanalabs/common/device.c
@@ -321,6 +321,10 @@ static int device_early_init(struct hl_device *hdev)
goto free_chip_info;
}
+ rc = hl_mmu_if_set_funcs(hdev);
+ if (rc)
+ goto free_idle_busy_ts_arr;
+
hl_cb_mgr_init(&hdev->kernel_cb_mgr);
mutex_init(&hdev->send_cpu_message_lock);
@@ -334,6 +338,8 @@ static int device_early_init(struct hl_device *hdev)
return 0;
+free_idle_busy_ts_arr:
+ kfree(hdev->idle_busy_ts_arr);
free_chip_info:
kfree(hdev->hl_chip_info);
free_eq_wq:
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index ec765320159a..15e746a5fe35 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -1423,6 +1423,44 @@ struct hl_device_idle_busy_ts {
ktime_t busy_to_idle_ts;
};
+
+/**
+ * struct hl_mmu_priv - used for holding per-device mmu internal information.
+ * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops.
+ * @mmu_shadow_hop0: shadow array of hop0 tables.
+ */
+struct hl_mmu_priv {
+ struct gen_pool *mmu_pgt_pool;
+ void *mmu_shadow_hop0;
+};
+
+/**
+ * struct hl_mmu_funcs - Device related MMU functions.
+ * @init: initialize the MMU module.
+ * @fini: release the MMU module.
+ * @ctx_init: Initialize a context for using the MMU module.
+ * @ctx_fini: disable a ctx from using the mmu module.
+ * @map: maps a virtual address to physical address for a context.
+ * @unmap: unmap a virtual address of a context.
+ * @flush: flush all writes from all cores to reach device MMU.
+ * @swap_out: marks all mapping of the given context as swapped out.
+ * @swap_in: marks all mapping of the given context as swapped in.
+ */
+struct hl_mmu_funcs {
+ int (*init)(struct hl_device *hdev);
+ void (*fini)(struct hl_device *hdev);
+ int (*ctx_init)(struct hl_ctx *ctx);
+ void (*ctx_fini)(struct hl_ctx *ctx);
+ int (*map)(struct hl_ctx *ctx,
+ u64 virt_addr, u64 phys_addr, u32 page_size,
+ bool is_dram_addr);
+ int (*unmap)(struct hl_ctx *ctx,
+ u64 virt_addr, bool is_dram_addr);
+ void (*flush)(struct hl_ctx *ctx);
+ void (*swap_out)(struct hl_ctx *ctx);
+ void (*swap_in)(struct hl_ctx *ctx);
+};
+
/**
* struct hl_device - habanalabs device structure.
* @pdev: pointer to PCI device, can be NULL in case of simulator device.
@@ -1459,10 +1497,8 @@ struct hl_device_idle_busy_ts {
* @asic_prop: ASIC specific immutable properties.
* @asic_funcs: ASIC specific functions.
* @asic_specific: ASIC specific information to use only from ASIC files.
- * @mmu_pgt_pool: pool of available MMU hops.
* @vm: virtual memory manager for MMU.
* @mmu_cache_lock: protects MMU cache invalidation as it can serve one context.
- * @mmu_shadow_hop0: shadow mapping of the MMU hop 0 zone.
* @hwmon_dev: H/W monitor device.
* @pm_mng_profile: current power management profile.
* @hl_chip_info: ASIC's sensors information.
@@ -1480,6 +1516,8 @@ struct hl_device_idle_busy_ts {
* @idle_busy_ts_arr: array to hold time stamps of transitions from idle to busy
* and vice-versa
* @aggregated_cs_counters: aggregated cs counters among all contexts
+ * @mmu_priv: device-specific MMU data.
+ * @mmu_func: device-related MMU functions.
* @dram_used_mem: current DRAM memory consumption.
* @timeout_jiffies: device CS timeout value.
* @max_power: the max power of the device, as configured by the sysadmin. This
@@ -1564,10 +1602,8 @@ struct hl_device {
struct asic_fixed_properties asic_prop;
const struct hl_asic_funcs *asic_funcs;
void *asic_specific;
- struct gen_pool *mmu_pgt_pool;
struct hl_vm vm;
struct mutex mmu_cache_lock;
- void *mmu_shadow_hop0;
struct device *hwmon_dev;
enum hl_pm_mng_profile pm_mng_profile;
struct hwmon_chip_info *hl_chip_info;
@@ -1591,6 +1627,9 @@ struct hl_device {
struct hl_cs_counters aggregated_cs_counters;
+ struct hl_mmu_priv mmu_priv;
+ struct hl_mmu_funcs mmu_func;
+
atomic64_t dram_used_mem;
u64 timeout_jiffies;
u64 max_power;
@@ -1842,6 +1881,8 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
bool flush_pte);
void hl_mmu_swap_out(struct hl_ctx *ctx);
void hl_mmu_swap_in(struct hl_ctx *ctx);
+int hl_mmu_if_set_funcs(struct hl_device *hdev);
+void hl_mmu_v1_set_funcs(struct hl_device *hdev);
int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
void __iomem *dst);
diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu.c
new file mode 100644
index 000000000000..b5058798aeb9
--- /dev/null
+++ b/drivers/misc/habanalabs/common/mmu.c
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include <linux/slab.h>
+
+#include "habanalabs.h"
+
+static bool is_dram_va(struct hl_device *hdev, u64 virt_addr)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
+ prop->dmmu.start_addr,
+ prop->dmmu.end_addr);
+}
+
+/**
+ * hl_mmu_init() - initialize the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Create a pool of pages for pgt_infos.
+ * - Create a shadow table for pgt
+ *
+ * Return: 0 for success, non-zero for failure.
+ */
+int hl_mmu_init(struct hl_device *hdev)
+{
+ if (hdev->mmu_enable)
+ return hdev->mmu_func.init(hdev);
+
+ return 0;
+}
+
+/**
+ * hl_mmu_fini() - release the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Disable MMU in H/W.
+ * - Free the pgt_infos pool.
+ *
+ * All contexts should be freed before calling this function.
+ */
+void hl_mmu_fini(struct hl_device *hdev)
+{
+ if (hdev->mmu_enable)
+ hdev->mmu_func.fini(hdev);
+}
+
+/**
+ * hl_mmu_ctx_init() - initialize a context for using the MMU module.
+ * @ctx: pointer to the context structure to initialize.
+ *
+ * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
+ * page tables hops related to this context.
+ * Return: 0 on success, non-zero otherwise.
+ */
+int hl_mmu_ctx_init(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (hdev->mmu_enable)
+ return hdev->mmu_func.ctx_init(ctx);
+
+ return 0;
+}
+
+/*
+ * hl_mmu_ctx_fini - disable a ctx from using the mmu module
+ *
+ * @ctx: pointer to the context structure
+ *
+ * This function does the following:
+ * - Free any pgts which were not freed yet
+ * - Free the mutex
+ * - Free DRAM default page mapping hops
+ */
+void hl_mmu_ctx_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (hdev->mmu_enable)
+ hdev->mmu_func.ctx_fini(ctx);
+}
+
+/*
+ * hl_mmu_unmap - unmaps a virtual addr
+ *
+ * @ctx: pointer to the context structure
+ * @virt_addr: virt addr to map from
+ * @page_size: size of the page to unmap
+ * @flush_pte: whether to do a PCI flush
+ *
+ * This function does the following:
+ * - Check that the virt addr is mapped
+ * - Unmap the virt addr and frees pgts if possible
+ * - Returns 0 on success, -EINVAL if the given addr is not mapped
+ *
+ * Because this function changes the page tables in the device and because it
+ * changes the MMU hash, it must be protected by a lock.
+ * However, because it maps only a single page, the lock should be implemented
+ * in a higher level in order to protect the entire mapping of the memory area
+ *
+ * For optimization reasons PCI flush may be requested once after unmapping of
+ * large area.
+ */
+int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
+ bool flush_pte)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_mmu_properties *mmu_prop;
+ u64 real_virt_addr;
+ u32 real_page_size, npages;
+ int i, rc = 0;
+ bool is_dram_addr;
+
+ if (!hdev->mmu_enable)
+ return 0;
+
+ is_dram_addr = is_dram_va(hdev, virt_addr);
+
+ if (is_dram_addr)
+ mmu_prop = &prop->dmmu;
+ else if ((page_size % prop->pmmu_huge.page_size) == 0)
+ mmu_prop = &prop->pmmu_huge;
+ else
+ mmu_prop = &prop->pmmu;
+
+ /*
+ * The H/W handles mapping of specific page sizes. Hence if the page
+ * size is bigger, we break it to sub-pages and unmap them separately.
+ */
+ if ((page_size % mmu_prop->page_size) == 0) {
+ real_page_size = mmu_prop->page_size;
+ } else {
+ dev_err(hdev->dev,
+ "page size of %u is not %uKB aligned, can't unmap\n",
+ page_size, mmu_prop->page_size >> 10);
+
+ return -EFAULT;
+ }
+
+ npages = page_size / real_page_size;
+ real_virt_addr = virt_addr;
+
+ for (i = 0 ; i < npages ; i++) {
+ rc = hdev->mmu_func.unmap(ctx, real_virt_addr, is_dram_addr);
+ if (rc)
+ break;
+
+ real_virt_addr += real_page_size;
+ }
+
+ if (flush_pte)
+ hdev->mmu_func.flush(ctx);
+
+ return rc;
+}
+
+/*
+ * hl_mmu_map - maps a virtual addr to physical addr
+ *
+ * @ctx: pointer to the context structure
+ * @virt_addr: virt addr to map from
+ * @phys_addr: phys addr to map to
+ * @page_size: physical page size
+ * @flush_pte: whether to do a PCI flush
+ *
+ * This function does the following:
+ * - Check that the virt addr is not mapped
+ * - Allocate pgts as necessary in order to map the virt addr to the phys
+ * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
+ *
+ * Because this function changes the page tables in the device and because it
+ * changes the MMU hash, it must be protected by a lock.
+ * However, because it maps only a single page, the lock should be implemented
+ * in a higher level in order to protect the entire mapping of the memory area
+ *
+ * For optimization reasons PCI flush may be requested once after mapping of
+ * large area.
+ */
+int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
+ bool flush_pte)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_mmu_properties *mmu_prop;
+ u64 real_virt_addr, real_phys_addr;
+ u32 real_page_size, npages;
+ int i, rc, mapped_cnt = 0;
+ bool is_dram_addr;
+
+ if (!hdev->mmu_enable)
+ return 0;
+
+ is_dram_addr = is_dram_va(hdev, virt_addr);
+
+ if (is_dram_addr)
+ mmu_prop = &prop->dmmu;
+ else if ((page_size % prop->pmmu_huge.page_size) == 0)
+ mmu_prop = &prop->pmmu_huge;
+ else
+ mmu_prop = &prop->pmmu;
+
+ /*
+ * The H/W handles mapping of specific page sizes. Hence if the page
+ * size is bigger, we break it to sub-pages and map them separately.
+ */
+ if ((page_size % mmu_prop->page_size) == 0) {
+ real_page_size = mmu_prop->page_size;
+ } else {
+ dev_err(hdev->dev,
+ "page size of %u is not %uKB aligned, can't unmap\n",
+ page_size, mmu_prop->page_size >> 10);
+
+ return -EFAULT;
+ }
+
+ WARN_ONCE((phys_addr & (real_page_size - 1)),
+ "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size",
+ phys_addr, real_page_size);
+
+ npages = page_size / real_page_size;
+ real_virt_addr = virt_addr;
+ real_phys_addr = phys_addr;
+
+ for (i = 0 ; i < npages ; i++) {
+ rc = hdev->mmu_func.map(ctx, real_virt_addr, real_phys_addr,
+ real_page_size, is_dram_addr);
+ if (rc)
+ goto err;
+
+ real_virt_addr += real_page_size;
+ real_phys_addr += real_page_size;
+ mapped_cnt++;
+ }
+
+ if (flush_pte)
+ hdev->mmu_func.flush(ctx);
+
+ return 0;
+
+err:
+ real_virt_addr = virt_addr;
+ for (i = 0 ; i < mapped_cnt ; i++) {
+ if (hdev->mmu_func.unmap(ctx, real_virt_addr, is_dram_addr))
+ dev_warn_ratelimited(hdev->dev,
+ "failed to unmap va: 0x%llx\n", real_virt_addr);
+
+ real_virt_addr += real_page_size;
+ }
+
+ hdev->mmu_func.flush(ctx);
+
+ return rc;
+}
+
+/*
+ * hl_mmu_swap_out - marks all mapping of the given ctx as swapped out
+ *
+ * @ctx: pointer to the context structure
+ *
+ */
+void hl_mmu_swap_out(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (hdev->mmu_enable)
+ hdev->mmu_func.swap_out(ctx);
+}
+
+/*
+ * hl_mmu_swap_in - marks all mapping of the given ctx as swapped in
+ *
+ * @ctx: pointer to the context structure
+ *
+ */
+void hl_mmu_swap_in(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (hdev->mmu_enable)
+ hdev->mmu_func.swap_in(ctx);
+}
+
+int hl_mmu_if_set_funcs(struct hl_device *hdev)
+{
+ if (!hdev->mmu_enable)
+ return 0;
+
+ switch (hdev->asic_type) {
+ case ASIC_GOYA:
+ case ASIC_GAUDI:
+ hl_mmu_v1_set_funcs(hdev);
+ break;
+ default:
+ dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
+ hdev->asic_type);
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
diff --git a/drivers/misc/habanalabs/common/mmu_v1.c b/drivers/misc/habanalabs/common/mmu_v1.c
index 3fc0f497fab3..8d1eb5265419 100644
--- a/drivers/misc/habanalabs/common/mmu_v1.c
+++ b/drivers/misc/habanalabs/common/mmu_v1.c
@@ -29,7 +29,7 @@ static void _free_hop(struct hl_ctx *ctx, struct pgt_info *pgt_info)
{
struct hl_device *hdev = ctx->hdev;
- gen_pool_free(hdev->mmu_pgt_pool, pgt_info->phys_addr,
+ gen_pool_free(hdev->mmu_priv.mmu_pgt_pool, pgt_info->phys_addr,
hdev->asic_prop.mmu_hop_table_size);
hash_del(&pgt_info->node);
kfree((u64 *) (uintptr_t) pgt_info->shadow_addr);
@@ -54,7 +54,7 @@ static u64 alloc_hop(struct hl_ctx *ctx)
if (!pgt_info)
return ULLONG_MAX;
- phys_addr = (u64) gen_pool_alloc(hdev->mmu_pgt_pool,
+ phys_addr = (u64) gen_pool_alloc(hdev->mmu_priv.mmu_pgt_pool,
prop->mmu_hop_table_size);
if (!phys_addr) {
dev_err(hdev->dev, "failed to allocate page\n");
@@ -75,7 +75,8 @@ static u64 alloc_hop(struct hl_ctx *ctx)
return shadow_addr;
shadow_err:
- gen_pool_free(hdev->mmu_pgt_pool, phys_addr, prop->mmu_hop_table_size);
+ gen_pool_free(hdev->mmu_priv.mmu_pgt_pool, phys_addr,
+ prop->mmu_hop_table_size);
pool_add_err:
kfree(pgt_info);
@@ -90,11 +91,11 @@ static inline u64 get_phys_hop0_addr(struct hl_ctx *ctx)
static inline u64 get_hop0_addr(struct hl_ctx *ctx)
{
- return (u64) (uintptr_t) ctx->hdev->mmu_shadow_hop0 +
+ return (u64) (uintptr_t) ctx->hdev->mmu_priv.mmu_shadow_hop0 +
(ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
}
-static inline void flush(struct hl_ctx *ctx)
+static void flush(struct hl_ctx *ctx)
{
/* flush all writes from all cores to reach PCI */
mb();
@@ -254,15 +255,6 @@ static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr)
return phys_hop_addr + pte_offset;
}
-static bool is_dram_va(struct hl_device *hdev, u64 virt_addr)
-{
- struct asic_fixed_properties *prop = &hdev->asic_prop;
-
- return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
- prop->dmmu.start_addr,
- prop->dmmu.end_addr);
-}
-
static int dram_default_mapping_init(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
@@ -413,7 +405,7 @@ static void dram_default_mapping_fini(struct hl_ctx *ctx)
}
/**
- * hl_mmu_init() - initialize the MMU module.
+ * hl_mmu_v1_init() - initialize the MMU module.
* @hdev: habanalabs device structure.
*
* This function does the following:
@@ -422,23 +414,20 @@ static void dram_default_mapping_fini(struct hl_ctx *ctx)
*
* Return: 0 for success, non-zero for failure.
*/
-int hl_mmu_init(struct hl_device *hdev)
+static int hl_mmu_v1_init(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
int rc;
- if (!hdev->mmu_enable)
- return 0;
-
- hdev->mmu_pgt_pool =
+ hdev->mmu_priv.mmu_pgt_pool =
gen_pool_create(__ffs(prop->mmu_hop_table_size), -1);
- if (!hdev->mmu_pgt_pool) {
+ if (!hdev->mmu_priv.mmu_pgt_pool) {
dev_err(hdev->dev, "Failed to create page gen pool\n");
return -ENOMEM;
}
- rc = gen_pool_add(hdev->mmu_pgt_pool, prop->mmu_pgt_addr +
+ rc = gen_pool_add(hdev->mmu_priv.mmu_pgt_pool, prop->mmu_pgt_addr +
prop->mmu_hop0_tables_total_size,
prop->mmu_pgt_size - prop->mmu_hop0_tables_total_size,
-1);
@@ -447,10 +436,10 @@ int hl_mmu_init(struct hl_device *hdev)
goto err_pool_add;
}
- hdev->mmu_shadow_hop0 = kvmalloc_array(prop->max_asid,
- prop->mmu_hop_table_size,
- GFP_KERNEL | __GFP_ZERO);
- if (ZERO_OR_NULL_PTR(hdev->mmu_shadow_hop0)) {
+ hdev->mmu_priv.mmu_shadow_hop0 = kvmalloc_array(prop->max_asid,
+ prop->mmu_hop_table_size,
+ GFP_KERNEL | __GFP_ZERO);
+ if (ZERO_OR_NULL_PTR(hdev->mmu_priv.mmu_shadow_hop0)) {
rc = -ENOMEM;
goto err_pool_add;
}
@@ -460,7 +449,7 @@ int hl_mmu_init(struct hl_device *hdev)
return 0;
err_pool_add:
- gen_pool_destroy(hdev->mmu_pgt_pool);
+ gen_pool_destroy(hdev->mmu_priv.mmu_pgt_pool);
return rc;
}
@@ -475,15 +464,12 @@ int hl_mmu_init(struct hl_device *hdev)
*
* All contexts should be freed before calling this function.
*/
-void hl_mmu_fini(struct hl_device *hdev)
+static void hl_mmu_v1_fini(struct hl_device *hdev)
{
- if (!hdev->mmu_enable)
- return;
-
/* MMU H/W fini was already done in device hw_fini() */
- kvfree(hdev->mmu_shadow_hop0);
- gen_pool_destroy(hdev->mmu_pgt_pool);
+ kvfree(hdev->mmu_priv.mmu_shadow_hop0);
+ gen_pool_destroy(hdev->mmu_priv.mmu_pgt_pool);
}
/**
@@ -494,13 +480,8 @@ void hl_mmu_fini(struct hl_device *hdev)
* page tables hops related to this context.
* Return: 0 on success, non-zero otherwise.
*/
-int hl_mmu_ctx_init(struct hl_ctx *ctx)
+static int hl_mmu_v1_ctx_init(struct hl_ctx *ctx)
{
- struct hl_device *hdev = ctx->hdev;
-
- if (!hdev->mmu_enable)
- return 0;
-
mutex_init(&ctx->mmu_lock);
hash_init(ctx->mmu_shadow_hash);
@@ -517,16 +498,13 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
* - Free the mutex
* - Free DRAM default page mapping hops
*/
-void hl_mmu_ctx_fini(struct hl_ctx *ctx)
+static void hl_mmu_v1_ctx_fini(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
struct pgt_info *pgt_info;
struct hlist_node *tmp;
int i;
- if (!hdev->mmu_enable)
- return;
-
dram_default_mapping_fini(ctx);
if (!hash_empty(ctx->mmu_shadow_hash))
@@ -543,7 +521,8 @@ void hl_mmu_ctx_fini(struct hl_ctx *ctx)
mutex_destroy(&ctx->mmu_lock);
}
-static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr)
+static int _hl_mmu_v1_unmap(struct hl_ctx *ctx,
+ u64 virt_addr, bool is_dram_addr)
{
struct hl_device *hdev = ctx->hdev;
struct asic_fixed_properties *prop = &hdev->asic_prop;
@@ -676,82 +655,7 @@ static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr)
return -EINVAL;
}
-/*
- * hl_mmu_unmap - unmaps a virtual addr
- *
- * @ctx: pointer to the context structure
- * @virt_addr: virt addr to map from
- * @page_size: size of the page to unmap
- * @flush_pte: whether to do a PCI flush
- *
- * This function does the following:
- * - Check that the virt addr is mapped
- * - Unmap the virt addr and frees pgts if possible
- * - Returns 0 on success, -EINVAL if the given addr is not mapped
- *
- * Because this function changes the page tables in the device and because it
- * changes the MMU hash, it must be protected by a lock.
- * However, because it maps only a single page, the lock should be implemented
- * in a higher level in order to protect the entire mapping of the memory area
- *
- * For optimization reasons PCI flush may be requested once after unmapping of
- * large area.
- */
-int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
- bool flush_pte)
-{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct hl_mmu_properties *mmu_prop;
- u64 real_virt_addr;
- u32 real_page_size, npages;
- int i, rc = 0;
- bool is_dram_addr;
-
- if (!hdev->mmu_enable)
- return 0;
-
- is_dram_addr = is_dram_va(hdev, virt_addr);
-
- if (is_dram_addr)
- mmu_prop = &prop->dmmu;
- else if ((page_size % prop->pmmu_huge.page_size) == 0)
- mmu_prop = &prop->pmmu_huge;
- else
- mmu_prop = &prop->pmmu;
-
- /*
- * The H/W handles mapping of specific page sizes. Hence if the page
- * size is bigger, we break it to sub-pages and unmap them separately.
- */
- if ((page_size % mmu_prop->page_size) == 0) {
- real_page_size = mmu_prop->page_size;
- } else {
- dev_err(hdev->dev,
- "page size of %u is not %uKB aligned, can't unmap\n",
- page_size, mmu_prop->page_size >> 10);
-
- return -EFAULT;
- }
-
- npages = page_size / real_page_size;
- real_virt_addr = virt_addr;
-
- for (i = 0 ; i < npages ; i++) {
- rc = _hl_mmu_unmap(ctx, real_virt_addr, is_dram_addr);
- if (rc)
- break;
-
- real_virt_addr += real_page_size;
- }
-
- if (flush_pte)
- flush(ctx);
-
- return rc;
-}
-
-static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
+static int _hl_mmu_v1_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
u32 page_size, bool is_dram_addr)
{
struct hl_device *hdev = ctx->hdev;
@@ -917,121 +821,43 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
}
/*
- * hl_mmu_map - maps a virtual addr to physical addr
+ * hl_mmu_v1_swap_out - marks all mapping of the given ctx as swapped out
*
* @ctx: pointer to the context structure
- * @virt_addr: virt addr to map from
- * @phys_addr: phys addr to map to
- * @page_size: physical page size
- * @flush_pte: whether to do a PCI flush
- *
- * This function does the following:
- * - Check that the virt addr is not mapped
- * - Allocate pgts as necessary in order to map the virt addr to the phys
- * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
- *
- * Because this function changes the page tables in the device and because it
- * changes the MMU hash, it must be protected by a lock.
- * However, because it maps only a single page, the lock should be implemented
- * in a higher level in order to protect the entire mapping of the memory area
*
- * For optimization reasons PCI flush may be requested once after mapping of
- * large area.
*/
-int hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
- bool flush_pte)
+static void hl_mmu_v1_swap_out(struct hl_ctx *ctx)
{
- struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct hl_mmu_properties *mmu_prop;
- u64 real_virt_addr, real_phys_addr;
- u32 real_page_size, npages;
- int i, rc, mapped_cnt = 0;
- bool is_dram_addr;
-
- if (!hdev->mmu_enable)
- return 0;
-
- is_dram_addr = is_dram_va(hdev, virt_addr);
-
- if (is_dram_addr)
- mmu_prop = &prop->dmmu;
- else if ((page_size % prop->pmmu_huge.page_size) == 0)
- mmu_prop = &prop->pmmu_huge;
- else
- mmu_prop = &prop->pmmu;
-
- /*
- * The H/W handles mapping of specific page sizes. Hence if the page
- * size is bigger, we break it to sub-pages and map them separately.
- */
- if ((page_size % mmu_prop->page_size) == 0) {
- real_page_size = mmu_prop->page_size;
- } else {
- dev_err(hdev->dev,
- "page size of %u is not %uKB aligned, can't unmap\n",
- page_size, mmu_prop->page_size >> 10);
- return -EFAULT;
- }
-
- WARN_ONCE((phys_addr & (real_page_size - 1)),
- "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size",
- phys_addr, real_page_size);
-
- npages = page_size / real_page_size;
- real_virt_addr = virt_addr;
- real_phys_addr = phys_addr;
-
- for (i = 0 ; i < npages ; i++) {
- rc = _hl_mmu_map(ctx, real_virt_addr, real_phys_addr,
- real_page_size, is_dram_addr);
- if (rc)
- goto err;
-
- real_virt_addr += real_page_size;
- real_phys_addr += real_page_size;
- mapped_cnt++;
- }
-
- if (flush_pte)
- flush(ctx);
-
- return 0;
-
-err:
- real_virt_addr = virt_addr;
- for (i = 0 ; i < mapped_cnt ; i++) {
- if (_hl_mmu_unmap(ctx, real_virt_addr, is_dram_addr))
- dev_warn_ratelimited(hdev->dev,
- "failed to unmap va: 0x%llx\n", real_virt_addr);
-
- real_virt_addr += real_page_size;
- }
-
- flush(ctx);
-
- return rc;
}
/*
- * hl_mmu_swap_out - marks all mapping of the given ctx as swapped out
+ * hl_mmu_v1_swap_in - marks all mapping of the given ctx as swapped in
*
* @ctx: pointer to the context structure
*
*/
-void hl_mmu_swap_out(struct hl_ctx *ctx)
+static void hl_mmu_v1_swap_in(struct hl_ctx *ctx)
{
}
/*
- * hl_mmu_swap_in - marks all mapping of the given ctx as swapped in
- *
- * @ctx: pointer to the context structure
+ * hl_mmu_v1_prepare - prepare mmu for working with mmu v1
*
+ * @hdev: pointer to the device structure
*/
-void hl_mmu_swap_in(struct hl_ctx *ctx)
-{
-
+void hl_mmu_v1_set_funcs(struct hl_device *hdev)
+{
+ struct hl_mmu_funcs *mmu = &hdev->mmu_func;
+
+ mmu->init = hl_mmu_v1_init;
+ mmu->fini = hl_mmu_v1_fini;
+ mmu->ctx_init = hl_mmu_v1_ctx_init;
+ mmu->ctx_fini = hl_mmu_v1_ctx_fini;
+ mmu->map = _hl_mmu_v1_map;
+ mmu->unmap = _hl_mmu_v1_unmap;
+ mmu->flush = flush;
+ mmu->swap_out = hl_mmu_v1_swap_out;
+ mmu->swap_in = hl_mmu_v1_swap_in;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 04/11] habanalabs: add num_hops to hl_mmu_properties
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
2020-09-10 18:15 ` [PATCH 02/11] habanalabs: rename mmu.c to mmu_v1.c Oded Gabbay
2020-09-10 18:15 ` [PATCH 03/11] habanalabs: refactor MMU as device-oriented Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 05/11] habanalabs: add debugfs support for MMU with 6 HOPs Oded Gabbay
` (6 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Moti Haimovski
From: Moti Haimovski <mhaimovski@habana.ai>
This commit adds the number of HOPs supported by the device to the
device MMU properties.
Signed-off-by: Moti Haimovski <mhaimovski@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/common/habanalabs.h | 6 ++++++
drivers/misc/habanalabs/gaudi/gaudi.c | 1 +
drivers/misc/habanalabs/goya/goya.c | 2 ++
drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h | 2 ++
4 files changed, 11 insertions(+)
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 15e746a5fe35..8ef61926ed3b 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -236,12 +236,15 @@ enum hl_device_hw_state {
* @hop2_shift: shift of hop 2 mask.
* @hop3_shift: shift of hop 3 mask.
* @hop4_shift: shift of hop 4 mask.
+ * @hop5_shift: shift of hop 5 mask.
* @hop0_mask: mask to get the PTE address in hop 0.
* @hop1_mask: mask to get the PTE address in hop 1.
* @hop2_mask: mask to get the PTE address in hop 2.
* @hop3_mask: mask to get the PTE address in hop 3.
* @hop4_mask: mask to get the PTE address in hop 4.
+ * @hop5_mask: mask to get the PTE address in hop 5.
* @page_size: default page size used to allocate memory.
+ * @num_hops: The amount of hops supported by the translation table.
*/
struct hl_mmu_properties {
u64 start_addr;
@@ -251,12 +254,15 @@ struct hl_mmu_properties {
u64 hop2_shift;
u64 hop3_shift;
u64 hop4_shift;
+ u64 hop5_shift;
u64 hop0_mask;
u64 hop1_mask;
u64 hop2_mask;
u64 hop3_mask;
u64 hop4_mask;
+ u64 hop5_mask;
u32 page_size;
+ u32 num_hops;
};
/**
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index 483989500863..bf010ff31ced 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -441,6 +441,7 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev)
prop->pmmu.end_addr =
(VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1;
prop->pmmu.page_size = PAGE_SIZE_4KB;
+ prop->pmmu.num_hops = MMU_ARCH_5_HOPS;
/* PMMU and HPMMU are the same except of page size */
memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 46a900fb3ef8..6f831cc54d37 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -426,12 +426,14 @@ int goya_get_fixed_properties(struct hl_device *hdev)
prop->dmmu.start_addr = VA_DDR_SPACE_START;
prop->dmmu.end_addr = VA_DDR_SPACE_END;
prop->dmmu.page_size = PAGE_SIZE_2MB;
+ prop->dmmu.num_hops = MMU_ARCH_5_HOPS;
/* shifts and masks are the same in PMMU and DMMU */
memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu));
prop->pmmu.start_addr = VA_HOST_SPACE_START;
prop->pmmu.end_addr = VA_HOST_SPACE_END;
prop->pmmu.page_size = PAGE_SIZE_4KB;
+ prop->pmmu.num_hops = MMU_ARCH_5_HOPS;
/* PMMU and HPMMU are the same except of page size */
memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
diff --git a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
index 468bb045fbd1..dedf20e8f956 100644
--- a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
+++ b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
@@ -29,6 +29,8 @@
#define HOP3_SHIFT 21
#define HOP4_SHIFT 12
+#define MMU_ARCH_5_HOPS 5
+
#define HOP_PHYS_ADDR_MASK (~FLAGS_MASK)
#define HL_PTE_SIZE sizeof(u64)
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 05/11] habanalabs: add debugfs support for MMU with 6 HOPs
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (2 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 04/11] habanalabs: add num_hops to hl_mmu_properties Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 06/11] habanalabs/gaudi: increase timeout for boot fit load Oded Gabbay
` (5 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Moti Haimovski
From: Moti Haimovski <mhaimovski@habana.ai>
This commit modify the existing debugfs code to support future devices that
have a 6 HOPs MMU implementation instead of 5 HOPs implementation.
Signed-off-by: Moti Haimovski <mhaimovski@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/common/debugfs.c | 70 ++++++++++++++++++++----
1 file changed, 60 insertions(+), 10 deletions(-)
diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c
index 9398a21e0398..7b1fb49c3501 100644
--- a/drivers/misc/habanalabs/common/debugfs.c
+++ b/drivers/misc/habanalabs/common/debugfs.c
@@ -354,6 +354,14 @@ static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx,
mmu_specs->hop4_shift);
}
+static inline u64 get_hop5_pte_addr(struct hl_ctx *ctx,
+ struct hl_mmu_properties *mmu_specs,
+ u64 hop_addr, u64 vaddr)
+{
+ return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop5_mask,
+ mmu_specs->hop5_shift);
+}
+
static inline u64 get_next_hop_addr(u64 curr_pte)
{
if (curr_pte & PAGE_PRESENT_MASK)
@@ -377,6 +385,7 @@ static int mmu_show(struct seq_file *s, void *data)
hop2_addr = 0, hop2_pte_addr = 0, hop2_pte = 0,
hop3_addr = 0, hop3_pte_addr = 0, hop3_pte = 0,
hop4_addr = 0, hop4_pte_addr = 0, hop4_pte = 0,
+ hop5_addr = 0, hop5_pte_addr = 0, hop5_pte = 0,
virt_addr = dev_entry->mmu_addr;
if (!hdev->mmu_enable)
@@ -428,20 +437,49 @@ static int mmu_show(struct seq_file *s, void *data)
hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);
hop3_pte = hdev->asic_funcs->read_pte(hdev, hop3_pte_addr);
- if (!(hop3_pte & LAST_MASK)) {
+ if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {
+ if (!(hop3_pte & LAST_MASK)) {
+ hop4_addr = get_next_hop_addr(hop3_pte);
+
+ if (hop4_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,
+ hop4_addr, virt_addr);
+ hop4_pte = hdev->asic_funcs->read_pte(hdev,
+ hop4_pte_addr);
+ if (!(hop4_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ } else {
+ if (!(hop3_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ }
+ } else {
hop4_addr = get_next_hop_addr(hop3_pte);
if (hop4_addr == ULLONG_MAX)
goto not_mapped;
- hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop4_addr,
- virt_addr);
- hop4_pte = hdev->asic_funcs->read_pte(hdev, hop4_pte_addr);
- if (!(hop4_pte & PAGE_PRESENT_MASK))
- goto not_mapped;
- } else {
- if (!(hop3_pte & PAGE_PRESENT_MASK))
- goto not_mapped;
+ hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,
+ hop4_addr, virt_addr);
+ hop4_pte = hdev->asic_funcs->read_pte(hdev,
+ hop4_pte_addr);
+ if (!(hop4_pte & LAST_MASK)) {
+ hop5_addr = get_next_hop_addr(hop4_pte);
+
+ if (hop5_addr == ULLONG_MAX)
+ goto not_mapped;
+
+ hop5_pte_addr = get_hop5_pte_addr(ctx, mmu_prop,
+ hop5_addr, virt_addr);
+ hop5_pte = hdev->asic_funcs->read_pte(hdev,
+ hop5_pte_addr);
+ if (!(hop5_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ } else {
+ if (!(hop4_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+ }
}
seq_printf(s, "asid: %u, virt_addr: 0x%llx\n",
@@ -463,10 +501,22 @@ static int mmu_show(struct seq_file *s, void *data)
seq_printf(s, "hop3_pte_addr: 0x%llx\n", hop3_pte_addr);
seq_printf(s, "hop3_pte: 0x%llx\n", hop3_pte);
- if (!(hop3_pte & LAST_MASK)) {
+ if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {
+ if (!(hop3_pte & LAST_MASK)) {
+ seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);
+ seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);
+ seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);
+ }
+ } else {
seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);
seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);
seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);
+
+ if (!(hop4_pte & LAST_MASK)) {
+ seq_printf(s, "hop5_addr: 0x%llx\n", hop5_addr);
+ seq_printf(s, "hop5_pte_addr: 0x%llx\n", hop5_pte_addr);
+ seq_printf(s, "hop5_pte: 0x%llx\n", hop5_pte);
+ }
}
goto out;
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 06/11] habanalabs/gaudi: increase timeout for boot fit load
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (3 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 05/11] habanalabs: add debugfs support for MMU with 6 HOPs Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 07/11] habanalabs: allow to wait on CS without sleep Oded Gabbay
` (4 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh
The firmware running in the boot stage takes more time to execute due to
increased security mechanisms. Therefore, we need to increase the timeout
we wait for the boot fit to finish loading.
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/gaudi/gaudiP.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h
index dd222bc128f9..b70b810c21c9 100644
--- a/drivers/misc/habanalabs/gaudi/gaudiP.h
+++ b/drivers/misc/habanalabs/gaudi/gaudiP.h
@@ -42,7 +42,7 @@
#define MAX_POWER_DEFAULT_PCI 200000 /* 200W */
#define MAX_POWER_DEFAULT_PMC 350000 /* 350W */
-#define GAUDI_CPU_TIMEOUT_USEC 15000000 /* 15s */
+#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */
#define TPC_ENABLED_MASK 0xFF
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 07/11] habanalabs: allow to wait on CS without sleep
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (4 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 06/11] habanalabs/gaudi: increase timeout for boot fit load Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 08/11] habanalabs: no need for DMA_SHARED_BUFFER Oded Gabbay
` (3 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh
The user sometimes wants to check if a CS has completed to clean resources.
In that case, the user doesn't want to sleep but just to check if the CS
has finished and continue with his code.
Add a new definition to the API of the wait on CS. The new definition says
that if the timeout is 0, the driver won't sleep at all but return
immediately after checking if the CS has finished.
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/common/command_submission.c | 7 +++++--
include/uapi/misc/habanalabs.h | 3 +++
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c
index 470bffbe9bdc..b2b974ecc431 100644
--- a/drivers/misc/habanalabs/common/command_submission.c
+++ b/drivers/misc/habanalabs/common/command_submission.c
@@ -1180,8 +1180,11 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
"Can't wait on CS %llu because current CS is at seq %llu\n",
seq, ctx->cs_sequence);
} else if (fence) {
- rc = wait_for_completion_interruptible_timeout(
- &fence->completion, timeout);
+ if (!timeout_us)
+ rc = completion_done(&fence->completion);
+ else
+ rc = wait_for_completion_interruptible_timeout(
+ &fence->completion, timeout);
if (fence->error == -ETIMEDOUT)
rc = -ETIMEDOUT;
diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h
index 69fb44d35292..d449f8a31ce6 100644
--- a/include/uapi/misc/habanalabs.h
+++ b/include/uapi/misc/habanalabs.h
@@ -914,6 +914,9 @@ struct hl_debug_args {
* inside the kernel until the CS has finished or until the user-requested
* timeout has expired.
*
+ * If the timeout value is 0, the driver won't sleep at all. It will check
+ * the status of the CS and return immediately
+ *
* The return value of the IOCTL is a standard Linux error code. The possible
* values are:
*
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 08/11] habanalabs: no need for DMA_SHARED_BUFFER
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (5 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 07/11] habanalabs: allow to wait on CS without sleep Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 09/11] habanalabs: Save context in a command buffer object Oded Gabbay
` (2 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh
Now that the driver no longer uses dma_buf, we can remove the select of
DMA_SHARED_BUFFER from kconfig.
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
drivers/misc/habanalabs/Kconfig | 1 -
1 file changed, 1 deletion(-)
diff --git a/drivers/misc/habanalabs/Kconfig b/drivers/misc/habanalabs/Kconfig
index 8eb5d38c618e..1640340d3e62 100644
--- a/drivers/misc/habanalabs/Kconfig
+++ b/drivers/misc/habanalabs/Kconfig
@@ -7,7 +7,6 @@ config HABANA_AI
tristate "HabanaAI accelerators (habanalabs)"
depends on PCI && HAS_IOMEM
select FRAME_VECTOR
- select DMA_SHARED_BUFFER
select GENERIC_ALLOCATOR
select HWMON
help
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 09/11] habanalabs: Save context in a command buffer object
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (6 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 08/11] habanalabs: no need for DMA_SHARED_BUFFER Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 10/11] habanalabs: Add an option to map CB to device MMU Oded Gabbay
2020-09-10 18:15 ` [PATCH 11/11] habanalabs: update firmware interface file Oded Gabbay
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Tomer Tayar
From: Tomer Tayar <ttayar@habana.ai>
Future changes require using a context while handling a command buffer,
and thus need to save the context in the command buffer object.
Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
.../misc/habanalabs/common/command_buffer.c | 22 +++++++++++--------
drivers/misc/habanalabs/common/debugfs.c | 2 +-
drivers/misc/habanalabs/common/habanalabs.h | 9 ++++----
drivers/misc/habanalabs/gaudi/gaudi.c | 10 +++++----
drivers/misc/habanalabs/goya/goya.c | 10 +++++----
5 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c
index ba63cee74050..0cb556fb4a8b 100644
--- a/drivers/misc/habanalabs/common/command_buffer.c
+++ b/drivers/misc/habanalabs/common/command_buffer.c
@@ -47,6 +47,8 @@ static void cb_release(struct kref *ref)
hl_debugfs_remove_cb(cb);
+ hl_ctx_put(cb->ctx);
+
cb_do_release(hdev, cb);
}
@@ -107,11 +109,12 @@ static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
}
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
- u32 cb_size, u64 *handle, int ctx_id, bool internal_cb)
+ struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
+ u64 *handle)
{
struct hl_cb *cb;
bool alloc_new_cb = true;
- int rc;
+ int rc, ctx_id = ctx->asid;
/*
* Can't use generic function to check this because of special case
@@ -163,7 +166,8 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
}
cb->hdev = hdev;
- cb->ctx_id = ctx_id;
+ cb->ctx = ctx;
+ hl_ctx_get(hdev, cb->ctx);
spin_lock(&mgr->cb_lock);
rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
@@ -191,6 +195,7 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
return 0;
release_cb:
+ hl_ctx_put(cb->ctx);
cb_do_release(hdev, cb);
out_err:
*handle = 0;
@@ -250,9 +255,8 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
args->in.cb_size, HL_MAX_CB_SIZE);
rc = -EINVAL;
} else {
- rc = hl_cb_create(hdev, &hpriv->cb_mgr,
- args->in.cb_size, &handle,
- hpriv->ctx->asid, false);
+ rc = hl_cb_create(hdev, &hpriv->cb_mgr, hpriv->ctx,
+ args->in.cb_size, false, &handle);
}
memset(args, 0, sizeof(*args));
@@ -424,7 +428,7 @@ void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
if (kref_put(&cb->refcount, cb_release) != 1)
dev_err(hdev->dev,
"CB %d for CTX ID %d is still alive\n",
- id, cb->ctx_id);
+ id, cb->ctx->asid);
}
idr_destroy(&mgr->cb_handles);
@@ -437,8 +441,8 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
struct hl_cb *cb;
int rc;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
- HL_KERNEL_ASID_ID, internal_cb);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, cb_size,
+ internal_cb, &cb_handle);
if (rc) {
dev_err(hdev->dev,
"Failed to allocate CB for the kernel driver %d\n", rc);
diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c
index 7b1fb49c3501..569d054a6b06 100644
--- a/drivers/misc/habanalabs/common/debugfs.c
+++ b/drivers/misc/habanalabs/common/debugfs.c
@@ -111,7 +111,7 @@ static int command_buffers_show(struct seq_file *s, void *data)
}
seq_printf(s,
" %03llu %d 0x%08x %d %d %d\n",
- cb->id, cb->ctx_id, cb->size,
+ cb->id, cb->ctx->asid, cb->size,
kref_read(&cb->refcount),
cb->mmap, cb->cs_cnt);
}
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 8ef61926ed3b..88c68b664ef6 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -417,6 +417,7 @@ struct hl_cb_mgr {
* struct hl_cb - describes a Command Buffer.
* @refcount: reference counter for usage of the CB.
* @hdev: pointer to device this CB belongs to.
+ * @ctx: pointer to the CB owner's context.
* @lock: spinlock to protect mmap/cs flows.
* @debugfs_list: node in debugfs list of command buffers.
* @pool_list: node in pool list of command buffers.
@@ -426,7 +427,6 @@ struct hl_cb_mgr {
* @mmap_size: Holds the CB's size that was mmaped.
* @size: holds the CB's size.
* @cs_cnt: holds number of CS that this CB participates in.
- * @ctx_id: holds the ID of the owner's context.
* @mmap: true if the CB is currently mmaped to user.
* @is_pool: true if CB was acquired from the pool, false otherwise.
* @is_internal: internaly allocated
@@ -434,6 +434,7 @@ struct hl_cb_mgr {
struct hl_cb {
struct kref refcount;
struct hl_device *hdev;
+ struct hl_ctx *ctx;
spinlock_t lock;
struct list_head debugfs_list;
struct list_head pool_list;
@@ -443,7 +444,6 @@ struct hl_cb {
u32 mmap_size;
u32 size;
u32 cs_cnt;
- u32 ctx_id;
u8 mmap;
u8 is_pool;
u8 is_internal;
@@ -1838,8 +1838,9 @@ void hl_sysfs_fini(struct hl_device *hdev);
int hl_hwmon_init(struct hl_device *hdev);
void hl_hwmon_fini(struct hl_device *hdev);
-int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 cb_size,
- u64 *handle, int ctx_id, bool internal_cb);
+int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
+ struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
+ u64 *handle);
int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index bf010ff31ced..b51cc6c1d541 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -4114,8 +4114,9 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev,
parser->patched_cb_size = parser->user_cb_size +
sizeof(struct packet_msg_prot) * 2;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, parser->patched_cb_size,
- &patched_cb_handle, HL_KERNEL_ASID_ID, false);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
+ parser->patched_cb_size, false,
+ &patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
@@ -4187,8 +4188,9 @@ static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
if (rc)
goto free_userptr;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, parser->patched_cb_size,
- &patched_cb_handle, HL_KERNEL_ASID_ID, false);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
+ parser->patched_cb_size, false,
+ &patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
"Failed to allocate patched CB for DMA CS %d\n", rc);
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 6f831cc54d37..6c81a4b148de 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -3810,8 +3810,9 @@ static int goya_parse_cb_mmu(struct hl_device *hdev,
parser->patched_cb_size = parser->user_cb_size +
sizeof(struct packet_msg_prot) * 2;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, parser->patched_cb_size,
- &patched_cb_handle, HL_KERNEL_ASID_ID, false);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
+ parser->patched_cb_size, false,
+ &patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
@@ -3883,8 +3884,9 @@ static int goya_parse_cb_no_mmu(struct hl_device *hdev,
if (rc)
goto free_userptr;
- rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, parser->patched_cb_size,
- &patched_cb_handle, HL_KERNEL_ASID_ID, false);
+ rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
+ parser->patched_cb_size, false,
+ &patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
"Failed to allocate patched CB for DMA CS %d\n", rc);
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 10/11] habanalabs: Add an option to map CB to device MMU
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (7 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 09/11] habanalabs: Save context in a command buffer object Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
2020-09-10 18:15 ` [PATCH 11/11] habanalabs: update firmware interface file Oded Gabbay
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh, Tomer Tayar
From: Tomer Tayar <ttayar@habana.ai>
There are cases in which the device should access the host memory of a
CB through the device MMU, and thus this memory should be mapped.
The patch adds a flag to the CB IOCTL, in which a user can ask the
driver to perform the mapping when creating a CB.
The mapping is allowed only if a dedicated VA range was allocated for
the specific ASIC.
Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
.../misc/habanalabs/common/command_buffer.c | 197 +++++++++++++++++-
drivers/misc/habanalabs/common/context.c | 12 +-
drivers/misc/habanalabs/common/habanalabs.h | 20 +-
drivers/misc/habanalabs/gaudi/gaudi.c | 4 +-
drivers/misc/habanalabs/goya/goya.c | 4 +-
include/uapi/misc/habanalabs.h | 12 +-
6 files changed, 238 insertions(+), 11 deletions(-)
diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c
index 0cb556fb4a8b..8e23fba2557e 100644
--- a/drivers/misc/habanalabs/common/command_buffer.c
+++ b/drivers/misc/habanalabs/common/command_buffer.c
@@ -13,6 +13,132 @@
#include <linux/uaccess.h>
#include <linux/genalloc.h>
+static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct hl_vm_va_block *va_block, *tmp;
+ dma_addr_t bus_addr;
+ u64 virt_addr;
+ u32 page_size = prop->pmmu.page_size;
+ s32 offset;
+ int rc;
+
+ if (!hdev->supports_cb_mapping) {
+ dev_err_ratelimited(hdev->dev,
+ "Cannot map CB because no VA range is allocated for CB mapping\n");
+ return -EINVAL;
+ }
+
+ if (!hdev->mmu_enable) {
+ dev_err_ratelimited(hdev->dev,
+ "Cannot map CB because MMU is disabled\n");
+ return -EINVAL;
+ }
+
+ INIT_LIST_HEAD(&cb->va_block_list);
+
+ for (bus_addr = cb->bus_address;
+ bus_addr < cb->bus_address + cb->size;
+ bus_addr += page_size) {
+
+ virt_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, page_size);
+ if (!virt_addr) {
+ dev_err(hdev->dev,
+ "Failed to allocate device virtual address for CB\n");
+ rc = -ENOMEM;
+ goto err_va_pool_free;
+ }
+
+ va_block = kzalloc(sizeof(*va_block), GFP_KERNEL);
+ if (!va_block) {
+ rc = -ENOMEM;
+ gen_pool_free(ctx->cb_va_pool, virt_addr, page_size);
+ goto err_va_pool_free;
+ }
+
+ va_block->start = virt_addr;
+ va_block->end = virt_addr + page_size;
+ va_block->size = page_size;
+ list_add_tail(&va_block->node, &cb->va_block_list);
+ }
+
+ mutex_lock(&ctx->mmu_lock);
+
+ bus_addr = cb->bus_address;
+ offset = 0;
+ list_for_each_entry(va_block, &cb->va_block_list, node) {
+ rc = hl_mmu_map(ctx, va_block->start, bus_addr, va_block->size,
+ list_is_last(&va_block->node,
+ &cb->va_block_list));
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to map va %#llx to CB's pa %#llx\n",
+ va_block->start, bus_addr);
+ goto err_va_umap;
+ }
+
+ bus_addr += va_block->size;
+ offset += va_block->size;
+ }
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, false, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+ cb->is_mmu_mapped = true;
+
+ return 0;
+
+err_va_umap:
+ list_for_each_entry(va_block, &cb->va_block_list, node) {
+ if (offset <= 0)
+ break;
+ hl_mmu_unmap(ctx, va_block->start, va_block->size,
+ offset <= va_block->size);
+ offset -= va_block->size;
+ }
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+err_va_pool_free:
+ list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
+ gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
+ list_del(&va_block->node);
+ kfree(va_block);
+ }
+
+ return rc;
+}
+
+static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct hl_vm_va_block *va_block, *tmp;
+
+ mutex_lock(&ctx->mmu_lock);
+
+ list_for_each_entry(va_block, &cb->va_block_list, node)
+ if (hl_mmu_unmap(ctx, va_block->start, va_block->size,
+ list_is_last(&va_block->node,
+ &cb->va_block_list)))
+ dev_warn_ratelimited(hdev->dev,
+ "Failed to unmap CB's va 0x%llx\n",
+ va_block->start);
+
+ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
+
+ mutex_unlock(&ctx->mmu_lock);
+
+ list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
+ gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
+ list_del(&va_block->node);
+ kfree(va_block);
+ }
+}
+
static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
{
if (cb->is_internal)
@@ -47,6 +173,9 @@ static void cb_release(struct kref *ref)
hl_debugfs_remove_cb(cb);
+ if (cb->is_mmu_mapped)
+ cb_unmap_mem(cb->ctx, cb);
+
hl_ctx_put(cb->ctx);
cb_do_release(hdev, cb);
@@ -110,7 +239,7 @@ static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
- u64 *handle)
+ bool map_cb, u64 *handle)
{
struct hl_cb *cb;
bool alloc_new_cb = true;
@@ -169,13 +298,26 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
cb->ctx = ctx;
hl_ctx_get(hdev, cb->ctx);
+ if (map_cb) {
+ if (ctx_id == HL_KERNEL_ASID_ID) {
+ dev_err(hdev->dev,
+ "CB mapping is not supported for kernel context\n");
+ rc = -EINVAL;
+ goto release_cb;
+ }
+
+ rc = cb_map_mem(ctx, cb);
+ if (rc)
+ goto release_cb;
+ }
+
spin_lock(&mgr->cb_lock);
rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
spin_unlock(&mgr->cb_lock);
if (rc < 0) {
dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
- goto release_cb;
+ goto unmap_mem;
}
cb->id = (u64) rc;
@@ -194,6 +336,9 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
return 0;
+unmap_mem:
+ if (cb->is_mmu_mapped)
+ cb_unmap_mem(cb->ctx, cb);
release_cb:
hl_ctx_put(cb->ctx);
cb_do_release(hdev, cb);
@@ -256,7 +401,9 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
rc = -EINVAL;
} else {
rc = hl_cb_create(hdev, &hpriv->cb_mgr, hpriv->ctx,
- args->in.cb_size, false, &handle);
+ args->in.cb_size, false,
+ !!(args->in.flags & HL_CB_FLAGS_MAP),
+ &handle);
}
memset(args, 0, sizeof(*args));
@@ -442,7 +589,7 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
int rc;
rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, cb_size,
- internal_cb, &cb_handle);
+ internal_cb, false, &cb_handle);
if (rc) {
dev_err(hdev->dev,
"Failed to allocate CB for the kernel driver %d\n", rc);
@@ -498,3 +645,45 @@ int hl_cb_pool_fini(struct hl_device *hdev)
return 0;
}
+
+int hl_cb_va_pool_init(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int rc;
+
+ if (!hdev->supports_cb_mapping)
+ return 0;
+
+ ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1);
+ if (!ctx->cb_va_pool) {
+ dev_err(hdev->dev,
+ "Failed to create VA gen pool for CB mapping\n");
+ return -ENOMEM;
+ }
+
+ rc = gen_pool_add(ctx->cb_va_pool, prop->cb_va_start_addr,
+ prop->cb_va_end_addr - prop->cb_va_start_addr, -1);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to add memory to VA gen pool for CB mapping\n");
+ goto err_pool_destroy;
+ }
+
+ return 0;
+
+err_pool_destroy:
+ gen_pool_destroy(ctx->cb_va_pool);
+
+ return rc;
+}
+
+void hl_cb_va_pool_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (!hdev->supports_cb_mapping)
+ return;
+
+ gen_pool_destroy(ctx->cb_va_pool);
+}
diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c
index b168a9fce817..df8171a2226c 100644
--- a/drivers/misc/habanalabs/common/context.c
+++ b/drivers/misc/habanalabs/common/context.c
@@ -37,6 +37,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
if ((hdev->in_debug) && (hdev->compute_ctx == ctx))
hl_device_set_debug_mode(hdev, false);
+ hl_cb_va_pool_fini(ctx);
hl_vm_ctx_fini(ctx);
hl_asid_free(hdev, ctx->asid);
} else {
@@ -155,15 +156,24 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
goto err_asid_free;
}
+ rc = hl_cb_va_pool_init(ctx);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to init VA pool for mapped CB\n");
+ goto err_vm_ctx_fini;
+ }
+
rc = hdev->asic_funcs->ctx_init(ctx);
if (rc) {
dev_err(hdev->dev, "ctx_init failed\n");
- goto err_vm_ctx_fini;
+ goto err_cb_va_pool_fini;
}
}
return 0;
+err_cb_va_pool_fini:
+ hl_cb_va_pool_fini(ctx);
err_vm_ctx_fini:
hl_vm_ctx_fini(ctx);
err_asid_free:
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 88c68b664ef6..eaa9bf3f82a3 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -291,6 +291,10 @@ struct hl_mmu_properties {
* @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
* @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
* @mmu_dram_default_page_addr: DRAM default page physical address.
+ * @cb_va_start_addr: virtual start address of command buffers which are mapped
+ * to the device's MMU.
+ * @cb_va_end_addr: virtual end address of command buffers which are mapped to
+ * the device's MMU.
* @mmu_pgt_size: MMU page tables total size.
* @mmu_pte_size: PTE size in MMU page tables.
* @mmu_hop_table_size: MMU hop table size.
@@ -339,6 +343,8 @@ struct asic_fixed_properties {
u64 pcie_aux_dbi_reg_addr;
u64 mmu_pgt_addr;
u64 mmu_dram_default_page_addr;
+ u64 cb_va_start_addr;
+ u64 cb_va_end_addr;
u32 mmu_pgt_size;
u32 mmu_pte_size;
u32 mmu_hop_table_size;
@@ -421,6 +427,8 @@ struct hl_cb_mgr {
* @lock: spinlock to protect mmap/cs flows.
* @debugfs_list: node in debugfs list of command buffers.
* @pool_list: node in pool list of command buffers.
+ * @va_block_list: list of virtual addresses blocks of the CB if it is mapped to
+ * the device's MMU.
* @id: the CB's ID.
* @kernel_address: Holds the CB's kernel virtual address.
* @bus_address: Holds the CB's DMA address.
@@ -430,6 +438,7 @@ struct hl_cb_mgr {
* @mmap: true if the CB is currently mmaped to user.
* @is_pool: true if CB was acquired from the pool, false otherwise.
* @is_internal: internaly allocated
+ * @is_mmu_mapped: true if the CB is mapped to the device's MMU.
*/
struct hl_cb {
struct kref refcount;
@@ -438,6 +447,7 @@ struct hl_cb {
spinlock_t lock;
struct list_head debugfs_list;
struct list_head pool_list;
+ struct list_head va_block_list;
u64 id;
u64 kernel_address;
dma_addr_t bus_address;
@@ -447,6 +457,7 @@ struct hl_cb {
u8 mmap;
u8 is_pool;
u8 is_internal;
+ u8 is_mmu_mapped;
};
@@ -843,6 +854,8 @@ struct hl_va_range {
* @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
* MMU hash or walking the PGT requires talking this lock.
* @debugfs_list: node in debugfs list of contexts.
+ * @cb_va_pool: device VA pool for command buffers which are mapped to the
+ * device's MMU.
* @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
* to user so user could inquire about CS. It is used as
* index to cs_pending array.
@@ -874,6 +887,7 @@ struct hl_ctx {
struct mutex mmu_lock;
struct list_head debugfs_list;
struct hl_cs_counters cs_counters;
+ struct gen_pool *cb_va_pool;
u64 cs_sequence;
u64 *dram_default_hops;
spinlock_t cs_lock;
@@ -1574,6 +1588,7 @@ struct hl_mmu_funcs {
* @sync_stream_queue_idx: helper index for sync stream queues initialization.
* @supports_coresight: is CoreSight supported.
* @supports_soft_reset: is soft reset supported.
+ * @supports_cb_mapping: is mapping a CB to the device's MMU supported.
*/
struct hl_device {
struct pci_dev *pdev;
@@ -1673,6 +1688,7 @@ struct hl_device {
u8 sync_stream_queue_idx;
u8 supports_coresight;
u8 supports_soft_reset;
+ u8 supports_cb_mapping;
/* Parameters for bring-up */
u8 mmu_enable;
@@ -1840,7 +1856,7 @@ void hl_hwmon_fini(struct hl_device *hdev);
int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
- u64 *handle);
+ bool map_cb, u64 *handle);
int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle);
int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
@@ -1852,6 +1868,8 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
bool internal_cb);
int hl_cb_pool_init(struct hl_device *hdev);
int hl_cb_pool_fini(struct hl_device *hdev);
+int hl_cb_va_pool_init(struct hl_ctx *ctx);
+void hl_cb_va_pool_fini(struct hl_ctx *ctx);
void hl_cs_rollback_all(struct hl_device *hdev);
struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index b51cc6c1d541..6f7f6ad7a358 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -4115,7 +4115,7 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev,
sizeof(struct packet_msg_prot) * 2;
rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
- parser->patched_cb_size, false,
+ parser->patched_cb_size, false, false,
&patched_cb_handle);
if (rc) {
@@ -4189,7 +4189,7 @@ static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
goto free_userptr;
rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
- parser->patched_cb_size, false,
+ parser->patched_cb_size, false, false,
&patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 6c81a4b148de..5cddd46a8fb8 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -3811,7 +3811,7 @@ static int goya_parse_cb_mmu(struct hl_device *hdev,
sizeof(struct packet_msg_prot) * 2;
rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
- parser->patched_cb_size, false,
+ parser->patched_cb_size, false, false,
&patched_cb_handle);
if (rc) {
@@ -3885,7 +3885,7 @@ static int goya_parse_cb_no_mmu(struct hl_device *hdev,
goto free_userptr;
rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
- parser->patched_cb_size, false,
+ parser->patched_cb_size, false, false,
&patched_cb_handle);
if (rc) {
dev_err(hdev->dev,
diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h
index d449f8a31ce6..9705b8adb60c 100644
--- a/include/uapi/misc/habanalabs.h
+++ b/include/uapi/misc/habanalabs.h
@@ -462,6 +462,9 @@ struct hl_info_args {
/* 2MB minus 32 bytes for 2xMSG_PROT */
#define HL_MAX_CB_SIZE (0x200000 - 32)
+/* Indicates whether the command buffer should be mapped to the device's MMU */
+#define HL_CB_FLAGS_MAP 0x1
+
struct hl_cb_in {
/* Handle of CB or 0 if we want to create one */
__u64 cb_handle;
@@ -473,7 +476,8 @@ struct hl_cb_in {
__u32 cb_size;
/* Context ID - Currently not in use */
__u32 ctx_id;
- __u32 pad;
+ /* HL_CB_FLAGS_* */
+ __u32 flags;
};
struct hl_cb_out {
@@ -856,6 +860,12 @@ struct hl_debug_args {
* When creating a new CB, the IOCTL returns a handle of it, and the user-space
* process needs to use that handle to mmap the buffer so it can access them.
*
+ * In some instances, the device must access the command buffer through the
+ * device's MMU, and thus its memory should be mapped. In these cases, user can
+ * indicate the driver that such a mapping is required.
+ * The resulting device virtual address will be used internally by the driver,
+ * and won't be returned to user.
+ *
*/
#define HL_IOCTL_CB \
_IOWR('H', 0x02, union hl_cb_args)
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 11/11] habanalabs: update firmware interface file
2020-09-10 18:15 [PATCH 01/11] habanalabs: use smallest possible alignment for virtual addresses Oded Gabbay
` (8 preceding siblings ...)
2020-09-10 18:15 ` [PATCH 10/11] habanalabs: Add an option to map CB to device MMU Oded Gabbay
@ 2020-09-10 18:15 ` Oded Gabbay
9 siblings, 0 replies; 11+ messages in thread
From: Oded Gabbay @ 2020-09-10 18:15 UTC (permalink / raw)
To: linux-kernel, SW_Drivers; +Cc: gregkh
Add new packet to fetch PLL information from firmware. This will be needed
in the future when the driver won't be able to access the PLL registers
directly
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
---
.../misc/habanalabs/include/common/cpucp_if.h | 25 +++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h
index dcde440427b4..2a5c9cb3d505 100644
--- a/drivers/misc/habanalabs/include/common/cpucp_if.h
+++ b/drivers/misc/habanalabs/include/common/cpucp_if.h
@@ -213,6 +213,12 @@ enum pq_init_status {
* Trigger the reset_history property of a specified current sensor.
* The packet's arguments specify the desired sensor and the field to
* set.
+ *
+ * CPUCP_PACKET_PLL_REG_GET
+ * Fetch register of PLL from the required PLL IP.
+ * The packet's arguments specify the PLL IP and the register to get.
+ * Each register is 32-bit value which is returned in result field.
+ *
*/
enum cpucp_packet_id {
@@ -245,6 +251,7 @@ enum cpucp_packet_id {
CPUCP_PACKET_PCIE_THROUGHPUT_GET, /* internal */
CPUCP_PACKET_PCIE_REPLAY_CNT_GET, /* internal */
CPUCP_PACKET_TOTAL_ENERGY_GET, /* internal */
+ CPUCP_PACKET_PLL_REG_GET, /* internal */
};
#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5
@@ -279,6 +286,11 @@ struct cpucp_packet {
__u8 pad; /* unused */
};
+ struct {/* For PLL register fetch */
+ __le16 pll_type;
+ __le16 pll_reg;
+ };
+
/* For any general request */
__le32 index;
@@ -354,6 +366,19 @@ enum cpucp_pcie_throughput_attributes {
cpucp_pcie_throughput_rx
};
+enum cpucp_pll_reg_attributes {
+ cpucp_pll_nr_reg,
+ cpucp_pll_nf_reg,
+ cpucp_pll_od_reg,
+ cpucp_pll_div_factor_reg,
+ cpucp_pll_div_sel_reg
+};
+
+enum cpucp_pll_type_attributes {
+ cpucp_pll_cpu,
+ cpucp_pll_pci,
+};
+
/* Event Queue Packets */
struct eq_generic_event {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread