[PATCH 0/6] Fix the debugfs splat from mock selftests (v3)

[PATCH 0/6] Fix the debugfs splat from mock selftests (v3)

[Jason Ekstrand]

This patch series fixes a miscellaneous collection of bugs that all add up
to all our mock selftests throwing dmesg warnings in CI.  As can be seen
from "drm/i915: Use a table for i915_init/exit", it's especially fun since
those warnings don't always show up in the selftests but can show up in
other random IGTs depending on test execution order.

Jason Ekstrand (6):
  drm/i915: Call i915_globals_exit() after i915_pmu_exit()
  drm/i915: Call i915_globals_exit() if pci_register_device() fails
  drm/i915: Use a table for i915_init/exit (v2)
  drm/ttm: Force re-init if ttm_global_init() fails
  drm/ttm: Initialize debugfs from ttm_global_init()
  drm/i915: Make the kmem slab for i915_buddy_block a global

 drivers/gpu/drm/i915/i915_buddy.c             |  44 ++++++--
 drivers/gpu/drm/i915/i915_buddy.h             |   3 +-
 drivers/gpu/drm/i915/i915_globals.c           |   6 +-
 drivers/gpu/drm/i915/i915_pci.c               | 104 ++++++++++++------
 drivers/gpu/drm/i915/i915_perf.c              |   3 +-
 drivers/gpu/drm/i915/i915_perf.h              |   2 +-
 drivers/gpu/drm/i915/i915_pmu.c               |   4 +-
 drivers/gpu/drm/i915/i915_pmu.h               |   4 +-
 .../gpu/drm/i915/selftests/i915_selftest.c    |   4 +-
 drivers/gpu/drm/ttm/ttm_device.c              |  14 +++
 drivers/gpu/drm/ttm/ttm_module.c              |  16 ---
 11 files changed, 136 insertions(+), 68 deletions(-)

-- 
2.31.1

[PATCH 1/6] drm/i915: Call i915_globals_exit() after i915_pmu_exit()

[Jason Ekstrand]

We should tear down in the opposite order we set up.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
 drivers/gpu/drm/i915/i915_pci.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 67696d7522718..50ed93b03e582 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -1244,8 +1244,8 @@ static void __exit i915_exit(void)
 
 	i915_perf_sysctl_unregister();
 	pci_unregister_driver(&i915_pci_driver);
-	i915_globals_exit();
 	i915_pmu_exit();
+	i915_globals_exit();
 }
 
 module_init(i915_init);
-- 
2.31.1

[PATCH 2/6] drm/i915: Call i915_globals_exit() if pci_register_device() fails

[Jason Ekstrand]

In the unlikely event that pci_register_device() fails, we were tearing
down our PMU setup but not globals.  This leaves a bunch of memory slabs
lying around.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Fixes: 32eb6bcfdda9 ("drm/i915: Make request allocation caches global")
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
---
 drivers/gpu/drm/i915/i915_globals.c | 4 ++--
 drivers/gpu/drm/i915/i915_pci.c     | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
index 77f1911c463b8..87267e1d2ad92 100644
--- a/drivers/gpu/drm/i915/i915_globals.c
+++ b/drivers/gpu/drm/i915/i915_globals.c
@@ -138,7 +138,7 @@ void i915_globals_unpark(void)
 	atomic_inc(&active);
 }
 
-static void __exit __i915_globals_flush(void)
+static void __i915_globals_flush(void)
 {
 	atomic_inc(&active); /* skip shrinking */
 
@@ -148,7 +148,7 @@ static void __exit __i915_globals_flush(void)
 	atomic_dec(&active);
 }
 
-void __exit i915_globals_exit(void)
+void i915_globals_exit(void)
 {
 	GEM_BUG_ON(atomic_read(&active));
 
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 50ed93b03e582..4e627b57d31a2 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -1230,6 +1230,7 @@ static int __init i915_init(void)
 	err = pci_register_driver(&i915_pci_driver);
 	if (err) {
 		i915_pmu_exit();
+		i915_globals_exit();
 		return err;
 	}
 
-- 
2.31.1

[PATCH 3/6] drm/i915: Use a table for i915_init/exit (v2)

[Jason Ekstrand]

If the driver was not fully loaded, we may still have globals lying
around.  If we don't tear those down in i915_exit(), we'll leak a bunch
of memory slabs.  This can happen two ways: use_kms = false and if we've
run mock selftests.  In either case, we have an early exit from
i915_init which happens after i915_globals_init() and we need to clean
up those globals.

The mock selftests case is especially sticky.  The load isn't entirely
a no-op.  We actually do quite a bit inside those selftests including
allocating a bunch of mock objects and running tests on them.  Once all
those tests are complete, we exit early from i915_init().  Perviously,
i915_init() would return a non-zero error code on failure and a zero
error code on success.  In the success case, we would get to i915_exit()
and check i915_pci_driver.driver.owner to detect if i915_init exited early
and do nothing.  In the failure case, we would fail i915_init() but
there would be no opportunity to clean up globals.

The most annoying part is that you don't actually notice the failure as
part of the self-tests since leaking a bit of memory, while bad, doesn't
result in anything observable from userspace.  Instead, the next time we
load the driver (usually for next IGT test), i915_globals_init() gets
invoked again, we go to allocate a bunch of new memory slabs, those
implicitly create debugfs entries, and debugfs warns that we're trying
to create directories and files that already exist.  Since this all
happens as part of the next driver load, it shows up in the dmesg-warn
of whatever IGT test ran after the mock selftests.

While the obvious thing to do here might be to call i915_globals_exit()
after selftests, that's not actually safe.  The dma-buf selftests call
i915_gem_prime_export which creates a file.  We call dma_buf_put() on
the resulting dmabuf which calls fput() on the file.  However, fput()
isn't immediate and gets flushed right before syscall returns.  This
means that all the fput()s from the selftests don't happen until right
before the module load syscall used to fire off the selftests returns
which is after i915_init().  If we call i915_globals_exit() in
i915_init() after selftests, we end up freeing slabs out from under
objects which won't get released until fput() is flushed at the end of
the module load syscall.

The solution here is to let i915_init() return success early and detect
the early success in i915_exit() and only tear down globals and nothing
else.  This way the module loads successfully, regardless of the success
or failure of the tests.  Because we've not enumerated any PCI devices,
no device nodes are created and it's entirely useless from userspace.
The only thing the module does at that point is hold on to a bit of
memory until we unload it and i915_exit() is called.  Importantly, this
means that everything from our selftests has the ability to properly
flush out between i915_init() and i915_exit() because there is at least
one syscall boundary in between.

In order to handle all the delicate init/exit cases, we convert the
whole thing to a table of init/exit pairs and track the init status in
the new init_progress global.  This allows us to ensure that i915_exit()
always tears down exactly the things that i915_init() successfully
initialized.  We also allow early-exit of i915_init() without failure by
an init function returning > 0.  This is useful for nomodeset, and
selftests.  For the mock selftests, we convert them to always return 1
so we get the desired behavior of the driver always succeeding to load
the driver and then properly tearing down the partially loaded driver.

v2 (Tvrtko Ursulin):
 - Guard init_funcs[i].exit with GEM_BUG_ON(i >= ARRAY_SIZE(init_funcs))
v2 (Daniel Vetter):
 - Update the docstring for i915.mock_selftests

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
 drivers/gpu/drm/i915/i915_pci.c               | 105 ++++++++++++------
 drivers/gpu/drm/i915/i915_perf.c              |   3 +-
 drivers/gpu/drm/i915/i915_perf.h              |   2 +-
 drivers/gpu/drm/i915/i915_pmu.c               |   4 +-
 drivers/gpu/drm/i915/i915_pmu.h               |   4 +-
 .../gpu/drm/i915/selftests/i915_selftest.c    |   4 +-
 6 files changed, 82 insertions(+), 40 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 4e627b57d31a2..5f05cb1b5ac6b 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -1185,27 +1185,9 @@ static void i915_pci_shutdown(struct pci_dev *pdev)
 	i915_driver_shutdown(i915);
 }
 
-static struct pci_driver i915_pci_driver = {
-	.name = DRIVER_NAME,
-	.id_table = pciidlist,
-	.probe = i915_pci_probe,
-	.remove = i915_pci_remove,
-	.shutdown = i915_pci_shutdown,
-	.driver.pm = &i915_pm_ops,
-};
-
-static int __init i915_init(void)
+static int i915_check_nomodeset(void)
 {
 	bool use_kms = true;
-	int err;
-
-	err = i915_globals_init();
-	if (err)
-		return err;
-
-	err = i915_mock_selftests();
-	if (err)
-		return err > 0 ? 0 : err;
 
 	/*
 	 * Enable KMS by default, unless explicitly overriden by
@@ -1222,31 +1204,88 @@ static int __init i915_init(void)
 	if (!use_kms) {
 		/* Silently fail loading to not upset userspace. */
 		DRM_DEBUG_DRIVER("KMS disabled.\n");
-		return 0;
+		return 1;
 	}
 
-	i915_pmu_init();
+	return 0;
+}
 
-	err = pci_register_driver(&i915_pci_driver);
-	if (err) {
-		i915_pmu_exit();
-		i915_globals_exit();
-		return err;
+static struct pci_driver i915_pci_driver = {
+	.name = DRIVER_NAME,
+	.id_table = pciidlist,
+	.probe = i915_pci_probe,
+	.remove = i915_pci_remove,
+	.shutdown = i915_pci_shutdown,
+	.driver.pm = &i915_pm_ops,
+};
+
+static int i915_register_pci_driver(void)
+{
+	return pci_register_driver(&i915_pci_driver);
+}
+
+static void i915_unregister_pci_driver(void)
+{
+	pci_unregister_driver(&i915_pci_driver);
+}
+
+static const struct {
+   int (*init)(void);
+   void (*exit)(void);
+} init_funcs[] = {
+	{ i915_globals_init, i915_globals_exit },
+	{ i915_mock_selftests, NULL },
+	{ i915_check_nomodeset, NULL },
+	{ i915_pmu_init, i915_pmu_exit },
+	{ i915_register_pci_driver, i915_unregister_pci_driver },
+	{ i915_perf_sysctl_register, i915_perf_sysctl_unregister },
+};
+static int init_progress;
+
+static int __init i915_init(void)
+{
+	int err, i;
+
+	for (i = 0; i < ARRAY_SIZE(init_funcs); i++) {
+		err = init_funcs[i].init();
+		if (err < 0) {
+			while (i--) {
+				if (init_funcs[i].exit)
+					init_funcs[i].exit();
+			}
+			return err;
+		} else if (err > 0) {
+			/*
+			 * Early-exit success is reserved for things which
+			 * don't have an exit() function because we have no
+			 * idea how far they got or how to partially tear
+			 * them down.
+			 */
+			WARN_ON(init_funcs[i].exit);
+
+			/*
+			 * We don't want to advertise devices with an only
+			 * partially initialized driver.
+			 */
+			WARN_ON(i915_pci_driver.driver.owner);
+			break;
+		}
 	}
 
-	i915_perf_sysctl_register();
+	init_progress = i;
+
 	return 0;
 }
 
 static void __exit i915_exit(void)
 {
-	if (!i915_pci_driver.driver.owner)
-		return;
+	int i;
 
-	i915_perf_sysctl_unregister();
-	pci_unregister_driver(&i915_pci_driver);
-	i915_pmu_exit();
-	i915_globals_exit();
+	for (i = init_progress - 1; i >= 0; i--) {
+		GEM_BUG_ON(i >= ARRAY_SIZE(init_funcs));
+		if (init_funcs[i].exit)
+			init_funcs[i].exit();
+	}
 }
 
 module_init(i915_init);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index b4ec114a4698b..48ddb363b3bda 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -4483,9 +4483,10 @@ static int destroy_config(int id, void *p, void *data)
 	return 0;
 }
 
-void i915_perf_sysctl_register(void)
+int i915_perf_sysctl_register(void)
 {
 	sysctl_header = register_sysctl_table(dev_root);
+	return 0;
 }
 
 void i915_perf_sysctl_unregister(void)
diff --git a/drivers/gpu/drm/i915/i915_perf.h b/drivers/gpu/drm/i915/i915_perf.h
index 882fdd0a76800..1d1329e5af3ae 100644
--- a/drivers/gpu/drm/i915/i915_perf.h
+++ b/drivers/gpu/drm/i915/i915_perf.h
@@ -23,7 +23,7 @@ void i915_perf_fini(struct drm_i915_private *i915);
 void i915_perf_register(struct drm_i915_private *i915);
 void i915_perf_unregister(struct drm_i915_private *i915);
 int i915_perf_ioctl_version(void);
-void i915_perf_sysctl_register(void);
+int i915_perf_sysctl_register(void);
 void i915_perf_sysctl_unregister(void);
 
 int i915_perf_open_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index 34d37d46a1262..eca92076f31d2 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -1088,7 +1088,7 @@ static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 
 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
 
-void i915_pmu_init(void)
+int i915_pmu_init(void)
 {
 	int ret;
 
@@ -1101,6 +1101,8 @@ void i915_pmu_init(void)
 			  ret);
 	else
 		cpuhp_slot = ret;
+
+	return 0;
 }
 
 void i915_pmu_exit(void)
diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h
index 60f9595f902cd..449057648f39b 100644
--- a/drivers/gpu/drm/i915/i915_pmu.h
+++ b/drivers/gpu/drm/i915/i915_pmu.h
@@ -147,14 +147,14 @@ struct i915_pmu {
 };
 
 #ifdef CONFIG_PERF_EVENTS
-void i915_pmu_init(void);
+int i915_pmu_init(void);
 void i915_pmu_exit(void);
 void i915_pmu_register(struct drm_i915_private *i915);
 void i915_pmu_unregister(struct drm_i915_private *i915);
 void i915_pmu_gt_parked(struct drm_i915_private *i915);
 void i915_pmu_gt_unparked(struct drm_i915_private *i915);
 #else
-static inline void i915_pmu_init(void) {}
+static inline int i915_pmu_init(void) { return 0; }
 static inline void i915_pmu_exit(void) {}
 static inline void i915_pmu_register(struct drm_i915_private *i915) {}
 static inline void i915_pmu_unregister(struct drm_i915_private *i915) {}
diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c
index 1bc11c09faef5..484759c9409c0 100644
--- a/drivers/gpu/drm/i915/selftests/i915_selftest.c
+++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c
@@ -187,7 +187,7 @@ int i915_mock_selftests(void)
 	err = run_selftests(mock, NULL);
 	if (err) {
 		i915_selftest.mock = err;
-		return err;
+		return 1;
 	}
 
 	if (i915_selftest.mock < 0) {
@@ -430,7 +430,7 @@ module_param_named(st_timeout, i915_selftest.timeout_ms, uint, 0400);
 module_param_named(st_filter, i915_selftest.filter, charp, 0400);
 
 module_param_named_unsafe(mock_selftests, i915_selftest.mock, int, 0400);
-MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then exit module)");
+MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then leave dummy module)");
 
 module_param_named_unsafe(live_selftests, i915_selftest.live, int, 0400);
 MODULE_PARM_DESC(live_selftests, "Run selftests after driver initialisation on the live system (0:disabled [default], 1:run tests then continue, -1:run tests then exit module)");
-- 
2.31.1

[PATCH 4/6] drm/ttm: Force re-init if ttm_global_init() fails

[Jason Ekstrand]

If we have a failure, decrement the reference count so that the next
call to ttm_global_init() will actually do something instead of assume
everything is all set up.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Fixes: 62b53b37e4b1 ("drm/ttm: use a static ttm_bo_global instance")
Reviewed-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/ttm/ttm_device.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 5f31acec3ad76..519deea8e39b7 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -100,6 +100,8 @@ static int ttm_global_init(void)
 	debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root,
 				&glob->bo_count);
 out:
+	if (ret)
+		--ttm_glob_use_count;
 	mutex_unlock(&ttm_global_mutex);
 	return ret;
 }
-- 
2.31.1

Re: [Intel-gfx] [PATCH 4/6] drm/ttm: Force re-init if ttm_global_init() fails

[Daniel Vetter]

On Wed, Jul 21, 2021 at 10:23:56AM -0500, Jason Ekstrand wrote:
> If we have a failure, decrement the reference count so that the next
> call to ttm_global_init() will actually do something instead of assume
> everything is all set up.
> 
> Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> Fixes: 62b53b37e4b1 ("drm/ttm: use a static ttm_bo_global instance")
> Reviewed-by: Christian König <christian.koenig@amd.com>

This one is already in drm-misc-fixes.
-Daniel

> ---
>  drivers/gpu/drm/ttm/ttm_device.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> index 5f31acec3ad76..519deea8e39b7 100644
> --- a/drivers/gpu/drm/ttm/ttm_device.c
> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> @@ -100,6 +100,8 @@ static int ttm_global_init(void)
>  	debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root,
>  				&glob->bo_count);
>  out:
> +	if (ret)
> +		--ttm_glob_use_count;
>  	mutex_unlock(&ttm_global_mutex);
>  	return ret;
>  }
> -- 
> 2.31.1
> 
> _______________________________________________
> Intel-gfx mailing list
> Intel-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gfx

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

[PATCH 5/6] drm/ttm: Initialize debugfs from ttm_global_init()

[Jason Ekstrand]

We create a bunch of debugfs entries as a side-effect of
ttm_global_init() and then never clean them up.  This isn't usually a
problem because we free the whole debugfs directory on module unload.
However, if the global reference count ever goes to zero and then
ttm_global_init() is called again, we'll re-create those debugfs entries
and debugfs will complain in dmesg that we're creating entries that
already exist.  This patch fixes this problem by changing the lifetime
of the whole TTM debugfs directory to match that of the TTM global
state.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
---
 drivers/gpu/drm/ttm/ttm_device.c | 12 ++++++++++++
 drivers/gpu/drm/ttm/ttm_module.c | 16 ----------------
 2 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 519deea8e39b7..74e3b460132b3 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -44,6 +44,8 @@ static unsigned ttm_glob_use_count;
 struct ttm_global ttm_glob;
 EXPORT_SYMBOL(ttm_glob);
 
+struct dentry *ttm_debugfs_root;
+
 static void ttm_global_release(void)
 {
 	struct ttm_global *glob = &ttm_glob;
@@ -53,6 +55,7 @@ static void ttm_global_release(void)
 		goto out;
 
 	ttm_pool_mgr_fini();
+	debugfs_remove(ttm_debugfs_root);
 
 	__free_page(glob->dummy_read_page);
 	memset(glob, 0, sizeof(*glob));
@@ -73,6 +76,13 @@ static int ttm_global_init(void)
 
 	si_meminfo(&si);
 
+	ttm_debugfs_root = debugfs_create_dir("ttm", NULL);
+	if (IS_ERR(ttm_debugfs_root)) {
+		ret = PTR_ERR(ttm_debugfs_root);
+		ttm_debugfs_root = NULL;
+		goto out;
+	}
+
 	/* Limit the number of pages in the pool to about 50% of the total
 	 * system memory.
 	 */
@@ -100,6 +110,8 @@ static int ttm_global_init(void)
 	debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root,
 				&glob->bo_count);
 out:
+	if (ret && ttm_debugfs_root)
+		debugfs_remove(ttm_debugfs_root);
 	if (ret)
 		--ttm_glob_use_count;
 	mutex_unlock(&ttm_global_mutex);
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index 997c458f68a9a..7fcdef278c742 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -72,22 +72,6 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
 	return tmp;
 }
 
-struct dentry *ttm_debugfs_root;
-
-static int __init ttm_init(void)
-{
-	ttm_debugfs_root = debugfs_create_dir("ttm", NULL);
-	return 0;
-}
-
-static void __exit ttm_exit(void)
-{
-	debugfs_remove(ttm_debugfs_root);
-}
-
-module_init(ttm_init);
-module_exit(ttm_exit);
-
 MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse");
 MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)");
 MODULE_LICENSE("GPL and additional rights");
-- 
2.31.1

Re: [PATCH 5/6] drm/ttm: Initialize debugfs from ttm_global_init()

[Daniel Vetter]

On Wed, Jul 21, 2021 at 10:23:57AM -0500, Jason Ekstrand wrote:
> We create a bunch of debugfs entries as a side-effect of
> ttm_global_init() and then never clean them up.  This isn't usually a
> problem because we free the whole debugfs directory on module unload.
> However, if the global reference count ever goes to zero and then
> ttm_global_init() is called again, we'll re-create those debugfs entries
> and debugfs will complain in dmesg that we're creating entries that
> already exist.  This patch fixes this problem by changing the lifetime
> of the whole TTM debugfs directory to match that of the TTM global
> state.
> 
> Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>

I've pushed this to drm-misc-fixes since I think this entire slab debugfs
story started with 5.15-rc1, so good to get it sorted there.
-Daniel

> ---
>  drivers/gpu/drm/ttm/ttm_device.c | 12 ++++++++++++
>  drivers/gpu/drm/ttm/ttm_module.c | 16 ----------------
>  2 files changed, 12 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> index 519deea8e39b7..74e3b460132b3 100644
> --- a/drivers/gpu/drm/ttm/ttm_device.c
> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> @@ -44,6 +44,8 @@ static unsigned ttm_glob_use_count;
>  struct ttm_global ttm_glob;
>  EXPORT_SYMBOL(ttm_glob);
>  
> +struct dentry *ttm_debugfs_root;
> +
>  static void ttm_global_release(void)
>  {
>  	struct ttm_global *glob = &ttm_glob;
> @@ -53,6 +55,7 @@ static void ttm_global_release(void)
>  		goto out;
>  
>  	ttm_pool_mgr_fini();
> +	debugfs_remove(ttm_debugfs_root);
>  
>  	__free_page(glob->dummy_read_page);
>  	memset(glob, 0, sizeof(*glob));
> @@ -73,6 +76,13 @@ static int ttm_global_init(void)
>  
>  	si_meminfo(&si);
>  
> +	ttm_debugfs_root = debugfs_create_dir("ttm", NULL);
> +	if (IS_ERR(ttm_debugfs_root)) {
> +		ret = PTR_ERR(ttm_debugfs_root);
> +		ttm_debugfs_root = NULL;
> +		goto out;
> +	}
> +
>  	/* Limit the number of pages in the pool to about 50% of the total
>  	 * system memory.
>  	 */
> @@ -100,6 +110,8 @@ static int ttm_global_init(void)
>  	debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root,
>  				&glob->bo_count);
>  out:
> +	if (ret && ttm_debugfs_root)
> +		debugfs_remove(ttm_debugfs_root);
>  	if (ret)
>  		--ttm_glob_use_count;
>  	mutex_unlock(&ttm_global_mutex);
> diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
> index 997c458f68a9a..7fcdef278c742 100644
> --- a/drivers/gpu/drm/ttm/ttm_module.c
> +++ b/drivers/gpu/drm/ttm/ttm_module.c
> @@ -72,22 +72,6 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
>  	return tmp;
>  }
>  
> -struct dentry *ttm_debugfs_root;
> -
> -static int __init ttm_init(void)
> -{
> -	ttm_debugfs_root = debugfs_create_dir("ttm", NULL);
> -	return 0;
> -}
> -
> -static void __exit ttm_exit(void)
> -{
> -	debugfs_remove(ttm_debugfs_root);
> -}
> -
> -module_init(ttm_init);
> -module_exit(ttm_exit);
> -
>  MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse");
>  MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)");
>  MODULE_LICENSE("GPL and additional rights");
> -- 
> 2.31.1
> 

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

[PATCH 6/6] drm/i915: Make the kmem slab for i915_buddy_block a global

[Jason Ekstrand]

There's no reason that I can tell why this should be per-i915_buddy_mm
and doing so causes KMEM_CACHE to throw dmesg warnings because it tries
to create a debugfs entry with the name i915_buddy_block multiple times.
We could handle this by carefully giving each slab its own name but that
brings its own pain because then we have to store that string somewhere
and manage the lifetimes of the different slabs.  The most likely
outcome would be a global atomic which we increment to get a new name or
something like that.

The much easier solution is to use the i915_globals system like we do
for every other slab in i915.  This ensures that we have exactly one of
them for each i915 driver load and it gets neatly created on module load
and destroyed on module unload.  Using the globals system also means
that its now tied into the shrink handler so we can properly respond to
low-memory situations.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Fixes: 88be9a0a06b7 ("drm/i915/ttm: add ttm_buddy_man")
Cc: Matthew Auld <matthew.auld@intel.com>
Cc: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/i915/i915_buddy.c   | 44 ++++++++++++++++++++++-------
 drivers/gpu/drm/i915/i915_buddy.h   |  3 +-
 drivers/gpu/drm/i915/i915_globals.c |  2 ++
 3 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c
index 29dd7d0310c1f..911feedad4513 100644
--- a/drivers/gpu/drm/i915/i915_buddy.c
+++ b/drivers/gpu/drm/i915/i915_buddy.c
@@ -8,8 +8,14 @@
 #include "i915_buddy.h"
 
 #include "i915_gem.h"
+#include "i915_globals.h"
 #include "i915_utils.h"
 
+static struct i915_global_buddy {
+	struct i915_global base;
+	struct kmem_cache *slab_blocks;
+} global;
+
 static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
 						 struct i915_buddy_block *parent,
 						 unsigned int order,
@@ -19,7 +25,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
 
 	GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
 
-	block = kmem_cache_zalloc(mm->slab_blocks, GFP_KERNEL);
+	block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
 	if (!block)
 		return NULL;
 
@@ -34,7 +40,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
 static void i915_block_free(struct i915_buddy_mm *mm,
 			    struct i915_buddy_block *block)
 {
-	kmem_cache_free(mm->slab_blocks, block);
+	kmem_cache_free(global.slab_blocks, block);
 }
 
 static void mark_allocated(struct i915_buddy_block *block)
@@ -85,15 +91,11 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
 
 	GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
 
-	mm->slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
-	if (!mm->slab_blocks)
-		return -ENOMEM;
-
 	mm->free_list = kmalloc_array(mm->max_order + 1,
 				      sizeof(struct list_head),
 				      GFP_KERNEL);
 	if (!mm->free_list)
-		goto out_destroy_slab;
+		return -ENOMEM;
 
 	for (i = 0; i <= mm->max_order; ++i)
 		INIT_LIST_HEAD(&mm->free_list[i]);
@@ -145,8 +147,6 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
 	kfree(mm->roots);
 out_free_list:
 	kfree(mm->free_list);
-out_destroy_slab:
-	kmem_cache_destroy(mm->slab_blocks);
 	return -ENOMEM;
 }
 
@@ -161,7 +161,6 @@ void i915_buddy_fini(struct i915_buddy_mm *mm)
 
 	kfree(mm->roots);
 	kfree(mm->free_list);
-	kmem_cache_destroy(mm->slab_blocks);
 }
 
 static int split_block(struct i915_buddy_mm *mm,
@@ -410,3 +409,28 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/i915_buddy.c"
 #endif
+
+static void i915_global_buddy_shrink(void)
+{
+	kmem_cache_shrink(global.slab_blocks);
+}
+
+static void i915_global_buddy_exit(void)
+{
+	kmem_cache_destroy(global.slab_blocks);
+}
+
+static struct i915_global_buddy global = { {
+	.shrink = i915_global_buddy_shrink,
+	.exit = i915_global_buddy_exit,
+} };
+
+int __init i915_global_buddy_init(void)
+{
+	global.slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
+	if (!global.slab_blocks)
+		return -ENOMEM;
+
+	i915_global_register(&global.base);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h
index 37f8c42071d12..d8f26706de52f 100644
--- a/drivers/gpu/drm/i915/i915_buddy.h
+++ b/drivers/gpu/drm/i915/i915_buddy.h
@@ -47,7 +47,6 @@ struct i915_buddy_block {
  * i915_buddy_alloc* and i915_buddy_free* should suffice.
  */
 struct i915_buddy_mm {
-	struct kmem_cache *slab_blocks;
 	/* Maintain a free list for each order. */
 	struct list_head *free_list;
 
@@ -130,4 +129,6 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
 
 void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
 
+int i915_global_buddy_init(void);
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
index 87267e1d2ad92..e57102a4c8d16 100644
--- a/drivers/gpu/drm/i915/i915_globals.c
+++ b/drivers/gpu/drm/i915/i915_globals.c
@@ -8,6 +8,7 @@
 #include <linux/workqueue.h>
 
 #include "i915_active.h"
+#include "i915_buddy.h"
 #include "gem/i915_gem_context.h"
 #include "gem/i915_gem_object.h"
 #include "i915_globals.h"
@@ -87,6 +88,7 @@ static void __i915_globals_cleanup(void)
 
 static __initconst int (* const initfn[])(void) = {
 	i915_global_active_init,
+	i915_global_buddy_init,
 	i915_global_context_init,
 	i915_global_gem_context_init,
 	i915_global_objects_init,
-- 
2.31.1

Re: [PATCH 6/6] drm/i915: Make the kmem slab for i915_buddy_block a global

[Matthew Auld]

On 21/07/2021 16:23, Jason Ekstrand wrote:
> There's no reason that I can tell why this should be per-i915_buddy_mm
> and doing so causes KMEM_CACHE to throw dmesg warnings because it tries
> to create a debugfs entry with the name i915_buddy_block multiple times.
> We could handle this by carefully giving each slab its own name but that
> brings its own pain because then we have to store that string somewhere
> and manage the lifetimes of the different slabs.  The most likely
> outcome would be a global atomic which we increment to get a new name or
> something like that.
> 
> The much easier solution is to use the i915_globals system like we do
> for every other slab in i915.  This ensures that we have exactly one of
> them for each i915 driver load and it gets neatly created on module load
> and destroyed on module unload.  Using the globals system also means
> that its now tied into the shrink handler so we can properly respond to
> low-memory situations.
> 
> Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> Fixes: 88be9a0a06b7 ("drm/i915/ttm: add ttm_buddy_man")
> Cc: Matthew Auld <matthew.auld@intel.com>
> Cc: Christian König <christian.koenig@amd.com>

It was intentionally ripped it out with the idea that we would be moving 
the buddy stuff into ttm, and so part of that was trying to get rid of 
the some of the i915 specifics, like this globals thing.

Reviewed-by: Matthew Auld <matthew.auld@intel.com>

> ---
>   drivers/gpu/drm/i915/i915_buddy.c   | 44 ++++++++++++++++++++++-------
>   drivers/gpu/drm/i915/i915_buddy.h   |  3 +-
>   drivers/gpu/drm/i915/i915_globals.c |  2 ++
>   3 files changed, 38 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c
> index 29dd7d0310c1f..911feedad4513 100644
> --- a/drivers/gpu/drm/i915/i915_buddy.c
> +++ b/drivers/gpu/drm/i915/i915_buddy.c
> @@ -8,8 +8,14 @@
>   #include "i915_buddy.h"
>   
>   #include "i915_gem.h"
> +#include "i915_globals.h"
>   #include "i915_utils.h"
>   
> +static struct i915_global_buddy {
> +	struct i915_global base;
> +	struct kmem_cache *slab_blocks;
> +} global;
> +
>   static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
>   						 struct i915_buddy_block *parent,
>   						 unsigned int order,
> @@ -19,7 +25,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
>   
>   	GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
>   
> -	block = kmem_cache_zalloc(mm->slab_blocks, GFP_KERNEL);
> +	block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
>   	if (!block)
>   		return NULL;
>   
> @@ -34,7 +40,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
>   static void i915_block_free(struct i915_buddy_mm *mm,
>   			    struct i915_buddy_block *block)
>   {
> -	kmem_cache_free(mm->slab_blocks, block);
> +	kmem_cache_free(global.slab_blocks, block);
>   }
>   
>   static void mark_allocated(struct i915_buddy_block *block)
> @@ -85,15 +91,11 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
>   
>   	GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
>   
> -	mm->slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
> -	if (!mm->slab_blocks)
> -		return -ENOMEM;
> -
>   	mm->free_list = kmalloc_array(mm->max_order + 1,
>   				      sizeof(struct list_head),
>   				      GFP_KERNEL);
>   	if (!mm->free_list)
> -		goto out_destroy_slab;
> +		return -ENOMEM;
>   
>   	for (i = 0; i <= mm->max_order; ++i)
>   		INIT_LIST_HEAD(&mm->free_list[i]);
> @@ -145,8 +147,6 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
>   	kfree(mm->roots);
>   out_free_list:
>   	kfree(mm->free_list);
> -out_destroy_slab:
> -	kmem_cache_destroy(mm->slab_blocks);
>   	return -ENOMEM;
>   }
>   
> @@ -161,7 +161,6 @@ void i915_buddy_fini(struct i915_buddy_mm *mm)
>   
>   	kfree(mm->roots);
>   	kfree(mm->free_list);
> -	kmem_cache_destroy(mm->slab_blocks);
>   }
>   
>   static int split_block(struct i915_buddy_mm *mm,
> @@ -410,3 +409,28 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
>   #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
>   #include "selftests/i915_buddy.c"
>   #endif
> +
> +static void i915_global_buddy_shrink(void)
> +{
> +	kmem_cache_shrink(global.slab_blocks);
> +}
> +
> +static void i915_global_buddy_exit(void)
> +{
> +	kmem_cache_destroy(global.slab_blocks);
> +}
> +
> +static struct i915_global_buddy global = { {
> +	.shrink = i915_global_buddy_shrink,
> +	.exit = i915_global_buddy_exit,
> +} };
> +
> +int __init i915_global_buddy_init(void)
> +{
> +	global.slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
> +	if (!global.slab_blocks)
> +		return -ENOMEM;
> +
> +	i915_global_register(&global.base);
> +	return 0;
> +}
> diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h
> index 37f8c42071d12..d8f26706de52f 100644
> --- a/drivers/gpu/drm/i915/i915_buddy.h
> +++ b/drivers/gpu/drm/i915/i915_buddy.h
> @@ -47,7 +47,6 @@ struct i915_buddy_block {
>    * i915_buddy_alloc* and i915_buddy_free* should suffice.
>    */
>   struct i915_buddy_mm {
> -	struct kmem_cache *slab_blocks;
>   	/* Maintain a free list for each order. */
>   	struct list_head *free_list;
>   
> @@ -130,4 +129,6 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
>   
>   void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
>   
> +int i915_global_buddy_init(void);
> +
>   #endif
> diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
> index 87267e1d2ad92..e57102a4c8d16 100644
> --- a/drivers/gpu/drm/i915/i915_globals.c
> +++ b/drivers/gpu/drm/i915/i915_globals.c
> @@ -8,6 +8,7 @@
>   #include <linux/workqueue.h>
>   
>   #include "i915_active.h"
> +#include "i915_buddy.h"
>   #include "gem/i915_gem_context.h"
>   #include "gem/i915_gem_object.h"
>   #include "i915_globals.h"
> @@ -87,6 +88,7 @@ static void __i915_globals_cleanup(void)
>   
>   static __initconst int (* const initfn[])(void) = {
>   	i915_global_active_init,
> +	i915_global_buddy_init,
>   	i915_global_context_init,
>   	i915_global_gem_context_init,
>   	i915_global_objects_init,
> 

Re: [Intel-gfx] [PATCH 6/6] drm/i915: Make the kmem slab for i915_buddy_block a global

[Daniel Vetter]

On Wed, Jul 21, 2021 at 05:25:41PM +0100, Matthew Auld wrote:
> On 21/07/2021 16:23, Jason Ekstrand wrote:
> > There's no reason that I can tell why this should be per-i915_buddy_mm
> > and doing so causes KMEM_CACHE to throw dmesg warnings because it tries
> > to create a debugfs entry with the name i915_buddy_block multiple times.
> > We could handle this by carefully giving each slab its own name but that
> > brings its own pain because then we have to store that string somewhere
> > and manage the lifetimes of the different slabs.  The most likely
> > outcome would be a global atomic which we increment to get a new name or
> > something like that.
> > 
> > The much easier solution is to use the i915_globals system like we do
> > for every other slab in i915.  This ensures that we have exactly one of
> > them for each i915 driver load and it gets neatly created on module load
> > and destroyed on module unload.  Using the globals system also means
> > that its now tied into the shrink handler so we can properly respond to
> > low-memory situations.
> > 
> > Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> > Fixes: 88be9a0a06b7 ("drm/i915/ttm: add ttm_buddy_man")
> > Cc: Matthew Auld <matthew.auld@intel.com>
> > Cc: Christian König <christian.koenig@amd.com>
> 
> It was intentionally ripped it out with the idea that we would be moving the
> buddy stuff into ttm, and so part of that was trying to get rid of the some
> of the i915 specifics, like this globals thing.
> 
> Reviewed-by: Matthew Auld <matthew.auld@intel.com>

I just sent out a patch to put i915_globals on a diet, so maybe we can
hold this patch here a bit when there's other reasons for why this is
special?

Or at least no make this use the i915_globals stuff and instead just link
up the init/exit function calls directly into Jason's new table, so that
we don't have a merge conflict here?
-Daniel

> 
> > ---
> >   drivers/gpu/drm/i915/i915_buddy.c   | 44 ++++++++++++++++++++++-------
> >   drivers/gpu/drm/i915/i915_buddy.h   |  3 +-
> >   drivers/gpu/drm/i915/i915_globals.c |  2 ++
> >   3 files changed, 38 insertions(+), 11 deletions(-)
> > 
> > diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c
> > index 29dd7d0310c1f..911feedad4513 100644
> > --- a/drivers/gpu/drm/i915/i915_buddy.c
> > +++ b/drivers/gpu/drm/i915/i915_buddy.c
> > @@ -8,8 +8,14 @@
> >   #include "i915_buddy.h"
> >   #include "i915_gem.h"
> > +#include "i915_globals.h"
> >   #include "i915_utils.h"
> > +static struct i915_global_buddy {
> > +	struct i915_global base;
> > +	struct kmem_cache *slab_blocks;
> > +} global;
> > +
> >   static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> >   						 struct i915_buddy_block *parent,
> >   						 unsigned int order,
> > @@ -19,7 +25,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> >   	GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
> > -	block = kmem_cache_zalloc(mm->slab_blocks, GFP_KERNEL);
> > +	block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
> >   	if (!block)
> >   		return NULL;
> > @@ -34,7 +40,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> >   static void i915_block_free(struct i915_buddy_mm *mm,
> >   			    struct i915_buddy_block *block)
> >   {
> > -	kmem_cache_free(mm->slab_blocks, block);
> > +	kmem_cache_free(global.slab_blocks, block);
> >   }
> >   static void mark_allocated(struct i915_buddy_block *block)
> > @@ -85,15 +91,11 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> >   	GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
> > -	mm->slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
> > -	if (!mm->slab_blocks)
> > -		return -ENOMEM;
> > -
> >   	mm->free_list = kmalloc_array(mm->max_order + 1,
> >   				      sizeof(struct list_head),
> >   				      GFP_KERNEL);
> >   	if (!mm->free_list)
> > -		goto out_destroy_slab;
> > +		return -ENOMEM;
> >   	for (i = 0; i <= mm->max_order; ++i)
> >   		INIT_LIST_HEAD(&mm->free_list[i]);
> > @@ -145,8 +147,6 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> >   	kfree(mm->roots);
> >   out_free_list:
> >   	kfree(mm->free_list);
> > -out_destroy_slab:
> > -	kmem_cache_destroy(mm->slab_blocks);
> >   	return -ENOMEM;
> >   }
> > @@ -161,7 +161,6 @@ void i915_buddy_fini(struct i915_buddy_mm *mm)
> >   	kfree(mm->roots);
> >   	kfree(mm->free_list);
> > -	kmem_cache_destroy(mm->slab_blocks);
> >   }
> >   static int split_block(struct i915_buddy_mm *mm,
> > @@ -410,3 +409,28 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
> >   #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
> >   #include "selftests/i915_buddy.c"
> >   #endif
> > +
> > +static void i915_global_buddy_shrink(void)
> > +{
> > +	kmem_cache_shrink(global.slab_blocks);
> > +}
> > +
> > +static void i915_global_buddy_exit(void)
> > +{
> > +	kmem_cache_destroy(global.slab_blocks);
> > +}
> > +
> > +static struct i915_global_buddy global = { {
> > +	.shrink = i915_global_buddy_shrink,
> > +	.exit = i915_global_buddy_exit,
> > +} };
> > +
> > +int __init i915_global_buddy_init(void)
> > +{
> > +	global.slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
> > +	if (!global.slab_blocks)
> > +		return -ENOMEM;
> > +
> > +	i915_global_register(&global.base);
> > +	return 0;
> > +}
> > diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h
> > index 37f8c42071d12..d8f26706de52f 100644
> > --- a/drivers/gpu/drm/i915/i915_buddy.h
> > +++ b/drivers/gpu/drm/i915/i915_buddy.h
> > @@ -47,7 +47,6 @@ struct i915_buddy_block {
> >    * i915_buddy_alloc* and i915_buddy_free* should suffice.
> >    */
> >   struct i915_buddy_mm {
> > -	struct kmem_cache *slab_blocks;
> >   	/* Maintain a free list for each order. */
> >   	struct list_head *free_list;
> > @@ -130,4 +129,6 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
> >   void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
> > +int i915_global_buddy_init(void);
> > +
> >   #endif
> > diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
> > index 87267e1d2ad92..e57102a4c8d16 100644
> > --- a/drivers/gpu/drm/i915/i915_globals.c
> > +++ b/drivers/gpu/drm/i915/i915_globals.c
> > @@ -8,6 +8,7 @@
> >   #include <linux/workqueue.h>
> >   #include "i915_active.h"
> > +#include "i915_buddy.h"
> >   #include "gem/i915_gem_context.h"
> >   #include "gem/i915_gem_object.h"
> >   #include "i915_globals.h"
> > @@ -87,6 +88,7 @@ static void __i915_globals_cleanup(void)
> >   static __initconst int (* const initfn[])(void) = {
> >   	i915_global_active_init,
> > +	i915_global_buddy_init,
> >   	i915_global_context_init,
> >   	i915_global_gem_context_init,
> >   	i915_global_objects_init,
> > 
> _______________________________________________
> Intel-gfx mailing list
> Intel-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gfx

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [Intel-gfx] [PATCH 6/6] drm/i915: Make the kmem slab for i915_buddy_block a global

[Jason Ekstrand]

On Wed, Jul 21, 2021 at 1:56 PM Daniel Vetter <daniel@ffwll.ch> wrote:
>
> On Wed, Jul 21, 2021 at 05:25:41PM +0100, Matthew Auld wrote:
> > On 21/07/2021 16:23, Jason Ekstrand wrote:
> > > There's no reason that I can tell why this should be per-i915_buddy_mm
> > > and doing so causes KMEM_CACHE to throw dmesg warnings because it tries
> > > to create a debugfs entry with the name i915_buddy_block multiple times.
> > > We could handle this by carefully giving each slab its own name but that
> > > brings its own pain because then we have to store that string somewhere
> > > and manage the lifetimes of the different slabs.  The most likely
> > > outcome would be a global atomic which we increment to get a new name or
> > > something like that.
> > >
> > > The much easier solution is to use the i915_globals system like we do
> > > for every other slab in i915.  This ensures that we have exactly one of
> > > them for each i915 driver load and it gets neatly created on module load
> > > and destroyed on module unload.  Using the globals system also means
> > > that its now tied into the shrink handler so we can properly respond to
> > > low-memory situations.
> > >
> > > Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> > > Fixes: 88be9a0a06b7 ("drm/i915/ttm: add ttm_buddy_man")
> > > Cc: Matthew Auld <matthew.auld@intel.com>
> > > Cc: Christian König <christian.koenig@amd.com>
> >
> > It was intentionally ripped it out with the idea that we would be moving the
> > buddy stuff into ttm, and so part of that was trying to get rid of the some
> > of the i915 specifics, like this globals thing.
> >
> > Reviewed-by: Matthew Auld <matthew.auld@intel.com>
>
> I just sent out a patch to put i915_globals on a diet, so maybe we can
> hold this patch here a bit when there's other reasons for why this is
> special?

This is required to get rid of the dmesg warnings.

> Or at least no make this use the i915_globals stuff and instead just link
> up the init/exit function calls directly into Jason's new table, so that
> we don't have a merge conflict here?

I'm happy to deal with merge conflicts however they land.

--Jason

> -Daniel
>
> >
> > > ---
> > >   drivers/gpu/drm/i915/i915_buddy.c   | 44 ++++++++++++++++++++++-------
> > >   drivers/gpu/drm/i915/i915_buddy.h   |  3 +-
> > >   drivers/gpu/drm/i915/i915_globals.c |  2 ++
> > >   3 files changed, 38 insertions(+), 11 deletions(-)
> > >
> > > diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c
> > > index 29dd7d0310c1f..911feedad4513 100644
> > > --- a/drivers/gpu/drm/i915/i915_buddy.c
> > > +++ b/drivers/gpu/drm/i915/i915_buddy.c
> > > @@ -8,8 +8,14 @@
> > >   #include "i915_buddy.h"
> > >   #include "i915_gem.h"
> > > +#include "i915_globals.h"
> > >   #include "i915_utils.h"
> > > +static struct i915_global_buddy {
> > > +   struct i915_global base;
> > > +   struct kmem_cache *slab_blocks;
> > > +} global;
> > > +
> > >   static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > >                                              struct i915_buddy_block *parent,
> > >                                              unsigned int order,
> > > @@ -19,7 +25,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > >     GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
> > > -   block = kmem_cache_zalloc(mm->slab_blocks, GFP_KERNEL);
> > > +   block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
> > >     if (!block)
> > >             return NULL;
> > > @@ -34,7 +40,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > >   static void i915_block_free(struct i915_buddy_mm *mm,
> > >                         struct i915_buddy_block *block)
> > >   {
> > > -   kmem_cache_free(mm->slab_blocks, block);
> > > +   kmem_cache_free(global.slab_blocks, block);
> > >   }
> > >   static void mark_allocated(struct i915_buddy_block *block)
> > > @@ -85,15 +91,11 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> > >     GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
> > > -   mm->slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
> > > -   if (!mm->slab_blocks)
> > > -           return -ENOMEM;
> > > -
> > >     mm->free_list = kmalloc_array(mm->max_order + 1,
> > >                                   sizeof(struct list_head),
> > >                                   GFP_KERNEL);
> > >     if (!mm->free_list)
> > > -           goto out_destroy_slab;
> > > +           return -ENOMEM;
> > >     for (i = 0; i <= mm->max_order; ++i)
> > >             INIT_LIST_HEAD(&mm->free_list[i]);
> > > @@ -145,8 +147,6 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> > >     kfree(mm->roots);
> > >   out_free_list:
> > >     kfree(mm->free_list);
> > > -out_destroy_slab:
> > > -   kmem_cache_destroy(mm->slab_blocks);
> > >     return -ENOMEM;
> > >   }
> > > @@ -161,7 +161,6 @@ void i915_buddy_fini(struct i915_buddy_mm *mm)
> > >     kfree(mm->roots);
> > >     kfree(mm->free_list);
> > > -   kmem_cache_destroy(mm->slab_blocks);
> > >   }
> > >   static int split_block(struct i915_buddy_mm *mm,
> > > @@ -410,3 +409,28 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
> > >   #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
> > >   #include "selftests/i915_buddy.c"
> > >   #endif
> > > +
> > > +static void i915_global_buddy_shrink(void)
> > > +{
> > > +   kmem_cache_shrink(global.slab_blocks);
> > > +}
> > > +
> > > +static void i915_global_buddy_exit(void)
> > > +{
> > > +   kmem_cache_destroy(global.slab_blocks);
> > > +}
> > > +
> > > +static struct i915_global_buddy global = { {
> > > +   .shrink = i915_global_buddy_shrink,
> > > +   .exit = i915_global_buddy_exit,
> > > +} };
> > > +
> > > +int __init i915_global_buddy_init(void)
> > > +{
> > > +   global.slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
> > > +   if (!global.slab_blocks)
> > > +           return -ENOMEM;
> > > +
> > > +   i915_global_register(&global.base);
> > > +   return 0;
> > > +}
> > > diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h
> > > index 37f8c42071d12..d8f26706de52f 100644
> > > --- a/drivers/gpu/drm/i915/i915_buddy.h
> > > +++ b/drivers/gpu/drm/i915/i915_buddy.h
> > > @@ -47,7 +47,6 @@ struct i915_buddy_block {
> > >    * i915_buddy_alloc* and i915_buddy_free* should suffice.
> > >    */
> > >   struct i915_buddy_mm {
> > > -   struct kmem_cache *slab_blocks;
> > >     /* Maintain a free list for each order. */
> > >     struct list_head *free_list;
> > > @@ -130,4 +129,6 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
> > >   void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
> > > +int i915_global_buddy_init(void);
> > > +
> > >   #endif
> > > diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
> > > index 87267e1d2ad92..e57102a4c8d16 100644
> > > --- a/drivers/gpu/drm/i915/i915_globals.c
> > > +++ b/drivers/gpu/drm/i915/i915_globals.c
> > > @@ -8,6 +8,7 @@
> > >   #include <linux/workqueue.h>
> > >   #include "i915_active.h"
> > > +#include "i915_buddy.h"
> > >   #include "gem/i915_gem_context.h"
> > >   #include "gem/i915_gem_object.h"
> > >   #include "i915_globals.h"
> > > @@ -87,6 +88,7 @@ static void __i915_globals_cleanup(void)
> > >   static __initconst int (* const initfn[])(void) = {
> > >     i915_global_active_init,
> > > +   i915_global_buddy_init,
> > >     i915_global_context_init,
> > >     i915_global_gem_context_init,
> > >     i915_global_objects_init,
> > >
> > _______________________________________________
> > Intel-gfx mailing list
> > Intel-gfx@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/intel-gfx
>
> --
> Daniel Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch

Re: [Intel-gfx] [PATCH 6/6] drm/i915: Make the kmem slab for i915_buddy_block a global

[Daniel Vetter]

On Wed, Jul 21, 2021 at 03:15:09PM -0500, Jason Ekstrand wrote:
> On Wed, Jul 21, 2021 at 1:56 PM Daniel Vetter <daniel@ffwll.ch> wrote:
> >
> > On Wed, Jul 21, 2021 at 05:25:41PM +0100, Matthew Auld wrote:
> > > On 21/07/2021 16:23, Jason Ekstrand wrote:
> > > > There's no reason that I can tell why this should be per-i915_buddy_mm
> > > > and doing so causes KMEM_CACHE to throw dmesg warnings because it tries
> > > > to create a debugfs entry with the name i915_buddy_block multiple times.
> > > > We could handle this by carefully giving each slab its own name but that
> > > > brings its own pain because then we have to store that string somewhere
> > > > and manage the lifetimes of the different slabs.  The most likely
> > > > outcome would be a global atomic which we increment to get a new name or
> > > > something like that.
> > > >
> > > > The much easier solution is to use the i915_globals system like we do
> > > > for every other slab in i915.  This ensures that we have exactly one of
> > > > them for each i915 driver load and it gets neatly created on module load
> > > > and destroyed on module unload.  Using the globals system also means
> > > > that its now tied into the shrink handler so we can properly respond to
> > > > low-memory situations.
> > > >
> > > > Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
> > > > Fixes: 88be9a0a06b7 ("drm/i915/ttm: add ttm_buddy_man")
> > > > Cc: Matthew Auld <matthew.auld@intel.com>
> > > > Cc: Christian König <christian.koenig@amd.com>
> > >
> > > It was intentionally ripped it out with the idea that we would be moving the
> > > buddy stuff into ttm, and so part of that was trying to get rid of the some
> > > of the i915 specifics, like this globals thing.
> > >
> > > Reviewed-by: Matthew Auld <matthew.auld@intel.com>
> >
> > I just sent out a patch to put i915_globals on a diet, so maybe we can
> > hold this patch here a bit when there's other reasons for why this is
> > special?
> 
> This is required to get rid of the dmesg warnings.
> 
> > Or at least no make this use the i915_globals stuff and instead just link
> > up the init/exit function calls directly into Jason's new table, so that
> > we don't have a merge conflict here?
> 
> I'm happy to deal with merge conflicts however they land.

I went hitshooting and rebased while applying :-)
-Daniel

> 
> --Jason
> 
> > -Daniel
> >
> > >
> > > > ---
> > > >   drivers/gpu/drm/i915/i915_buddy.c   | 44 ++++++++++++++++++++++-------
> > > >   drivers/gpu/drm/i915/i915_buddy.h   |  3 +-
> > > >   drivers/gpu/drm/i915/i915_globals.c |  2 ++
> > > >   3 files changed, 38 insertions(+), 11 deletions(-)
> > > >
> > > > diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c
> > > > index 29dd7d0310c1f..911feedad4513 100644
> > > > --- a/drivers/gpu/drm/i915/i915_buddy.c
> > > > +++ b/drivers/gpu/drm/i915/i915_buddy.c
> > > > @@ -8,8 +8,14 @@
> > > >   #include "i915_buddy.h"
> > > >   #include "i915_gem.h"
> > > > +#include "i915_globals.h"
> > > >   #include "i915_utils.h"
> > > > +static struct i915_global_buddy {
> > > > +   struct i915_global base;
> > > > +   struct kmem_cache *slab_blocks;
> > > > +} global;
> > > > +
> > > >   static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > > >                                              struct i915_buddy_block *parent,
> > > >                                              unsigned int order,
> > > > @@ -19,7 +25,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > > >     GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
> > > > -   block = kmem_cache_zalloc(mm->slab_blocks, GFP_KERNEL);
> > > > +   block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
> > > >     if (!block)
> > > >             return NULL;
> > > > @@ -34,7 +40,7 @@ static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
> > > >   static void i915_block_free(struct i915_buddy_mm *mm,
> > > >                         struct i915_buddy_block *block)
> > > >   {
> > > > -   kmem_cache_free(mm->slab_blocks, block);
> > > > +   kmem_cache_free(global.slab_blocks, block);
> > > >   }
> > > >   static void mark_allocated(struct i915_buddy_block *block)
> > > > @@ -85,15 +91,11 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> > > >     GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
> > > > -   mm->slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
> > > > -   if (!mm->slab_blocks)
> > > > -           return -ENOMEM;
> > > > -
> > > >     mm->free_list = kmalloc_array(mm->max_order + 1,
> > > >                                   sizeof(struct list_head),
> > > >                                   GFP_KERNEL);
> > > >     if (!mm->free_list)
> > > > -           goto out_destroy_slab;
> > > > +           return -ENOMEM;
> > > >     for (i = 0; i <= mm->max_order; ++i)
> > > >             INIT_LIST_HEAD(&mm->free_list[i]);
> > > > @@ -145,8 +147,6 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
> > > >     kfree(mm->roots);
> > > >   out_free_list:
> > > >     kfree(mm->free_list);
> > > > -out_destroy_slab:
> > > > -   kmem_cache_destroy(mm->slab_blocks);
> > > >     return -ENOMEM;
> > > >   }
> > > > @@ -161,7 +161,6 @@ void i915_buddy_fini(struct i915_buddy_mm *mm)
> > > >     kfree(mm->roots);
> > > >     kfree(mm->free_list);
> > > > -   kmem_cache_destroy(mm->slab_blocks);
> > > >   }
> > > >   static int split_block(struct i915_buddy_mm *mm,
> > > > @@ -410,3 +409,28 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
> > > >   #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
> > > >   #include "selftests/i915_buddy.c"
> > > >   #endif
> > > > +
> > > > +static void i915_global_buddy_shrink(void)
> > > > +{
> > > > +   kmem_cache_shrink(global.slab_blocks);
> > > > +}
> > > > +
> > > > +static void i915_global_buddy_exit(void)
> > > > +{
> > > > +   kmem_cache_destroy(global.slab_blocks);
> > > > +}
> > > > +
> > > > +static struct i915_global_buddy global = { {
> > > > +   .shrink = i915_global_buddy_shrink,
> > > > +   .exit = i915_global_buddy_exit,
> > > > +} };
> > > > +
> > > > +int __init i915_global_buddy_init(void)
> > > > +{
> > > > +   global.slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
> > > > +   if (!global.slab_blocks)
> > > > +           return -ENOMEM;
> > > > +
> > > > +   i915_global_register(&global.base);
> > > > +   return 0;
> > > > +}
> > > > diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h
> > > > index 37f8c42071d12..d8f26706de52f 100644
> > > > --- a/drivers/gpu/drm/i915/i915_buddy.h
> > > > +++ b/drivers/gpu/drm/i915/i915_buddy.h
> > > > @@ -47,7 +47,6 @@ struct i915_buddy_block {
> > > >    * i915_buddy_alloc* and i915_buddy_free* should suffice.
> > > >    */
> > > >   struct i915_buddy_mm {
> > > > -   struct kmem_cache *slab_blocks;
> > > >     /* Maintain a free list for each order. */
> > > >     struct list_head *free_list;
> > > > @@ -130,4 +129,6 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
> > > >   void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
> > > > +int i915_global_buddy_init(void);
> > > > +
> > > >   #endif
> > > > diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
> > > > index 87267e1d2ad92..e57102a4c8d16 100644
> > > > --- a/drivers/gpu/drm/i915/i915_globals.c
> > > > +++ b/drivers/gpu/drm/i915/i915_globals.c
> > > > @@ -8,6 +8,7 @@
> > > >   #include <linux/workqueue.h>
> > > >   #include "i915_active.h"
> > > > +#include "i915_buddy.h"
> > > >   #include "gem/i915_gem_context.h"
> > > >   #include "gem/i915_gem_object.h"
> > > >   #include "i915_globals.h"
> > > > @@ -87,6 +88,7 @@ static void __i915_globals_cleanup(void)
> > > >   static __initconst int (* const initfn[])(void) = {
> > > >     i915_global_active_init,
> > > > +   i915_global_buddy_init,
> > > >     i915_global_context_init,
> > > >     i915_global_gem_context_init,
> > > >     i915_global_objects_init,
> > > >
> > > _______________________________________________
> > > Intel-gfx mailing list
> > > Intel-gfx@lists.freedesktop.org
> > > https://lists.freedesktop.org/mailman/listinfo/intel-gfx
> >
> > --
> > Daniel Vetter
> > Software Engineer, Intel Corporation
> > http://blog.ffwll.ch

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch