* [PATCH] drm/ttm: add NUMA node id to the pool
@ 2023-03-30 19:50 Alex Deucher
2023-03-31 6:54 ` Christian König
0 siblings, 1 reply; 5+ messages in thread
From: Alex Deucher @ 2023-03-30 19:50 UTC (permalink / raw)
To: amd-gfx, dri-devel
Cc: Alex Deucher, Felix Kuehling, Rajneesh Bhardwaj, Graham Sider,
Christian König
From: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
This allows backing ttm_tt structure with pages from different NUMA
pools.
Tested-by: Graham Sider <graham.sider@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
---
drivers/gpu/drm/ttm/ttm_device.c | 2 +-
drivers/gpu/drm/ttm/ttm_pool.c | 13 ++++++++-----
include/drm/ttm/ttm_pool.h | 4 +++-
3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index e7147e304637..4a8164a5320f 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
bdev->funcs = funcs;
ttm_sys_man_init(bdev);
- ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
+ ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
bdev->vma_manager = vma_manager;
INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 9f6764bf3b15..1068a41cded1 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
__GFP_KSWAPD_RECLAIM;
if (!pool->use_dma_alloc) {
- p = alloc_pages(gfp_flags, order);
+ p = alloc_pages_node(pool->nid, gfp_flags, order);
if (p)
p->private = order;
return p;
@@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
enum ttm_caching caching,
unsigned int order)
{
- if (pool->use_dma_alloc)
+ if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
return &pool->caching[caching].orders[order];
#ifdef CONFIG_X86
@@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
*
* @pool: the pool to initialize
* @dev: device for DMA allocations and mappings
+ * @nid: NUMA node to use for allocations
* @use_dma_alloc: true if coherent DMA alloc should be used
* @use_dma32: true if GFP_DMA32 should be used
*
* Initialize the pool and its pool types.
*/
void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
- bool use_dma_alloc, bool use_dma32)
+ int nid, bool use_dma_alloc, bool use_dma32)
{
unsigned int i, j;
WARN_ON(!dev && use_dma_alloc);
pool->dev = dev;
+ pool->nid = nid;
pool->use_dma_alloc = use_dma_alloc;
pool->use_dma32 = use_dma32;
- if (use_dma_alloc) {
+ if (use_dma_alloc || nid != NUMA_NO_NODE) {
for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
for (j = 0; j < MAX_ORDER; ++j)
ttm_pool_type_init(&pool->caching[i].orders[j],
pool, i, j);
}
}
+EXPORT_SYMBOL(ttm_pool_init);
/**
* ttm_pool_fini - Cleanup a pool
@@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
{
unsigned int i, j;
- if (pool->use_dma_alloc) {
+ if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
for (j = 0; j < MAX_ORDER; ++j)
ttm_pool_type_fini(&pool->caching[i].orders[j]);
diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
index ef09b23d29e3..23bd8be6d4f8 100644
--- a/include/drm/ttm/ttm_pool.h
+++ b/include/drm/ttm/ttm_pool.h
@@ -61,12 +61,14 @@ struct ttm_pool_type {
* struct ttm_pool - Pool for all caching and orders
*
* @dev: the device we allocate pages for
+ * @nid: which numa node to use
* @use_dma_alloc: if coherent DMA allocations should be used
* @use_dma32: if GFP_DMA32 should be used
* @caching: pools for each caching/order
*/
struct ttm_pool {
struct device *dev;
+ int nid;
bool use_dma_alloc;
bool use_dma32;
@@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
- bool use_dma_alloc, bool use_dma32);
+ int nid, bool use_dma_alloc, bool use_dma32);
void ttm_pool_fini(struct ttm_pool *pool);
int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
--
2.39.2
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH] drm/ttm: add NUMA node id to the pool
2023-03-30 19:50 [PATCH] drm/ttm: add NUMA node id to the pool Alex Deucher
@ 2023-03-31 6:54 ` Christian König
2023-03-31 19:17 ` Alex Deucher
0 siblings, 1 reply; 5+ messages in thread
From: Christian König @ 2023-03-31 6:54 UTC (permalink / raw)
To: Alex Deucher, amd-gfx, dri-devel
Cc: Felix Kuehling, Rajneesh Bhardwaj, Graham Sider, Christian König
Should I push this to drm-misc-next or do we take it through
amd-staging-drm-next?
Christian.
Am 30.03.23 um 21:50 schrieb Alex Deucher:
> From: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
>
> This allows backing ttm_tt structure with pages from different NUMA
> pools.
>
> Tested-by: Graham Sider <graham.sider@amd.com>
> Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
> Signed-off-by: Christian König <christian.koenig@amd.com>
> Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> ---
> drivers/gpu/drm/ttm/ttm_device.c | 2 +-
> drivers/gpu/drm/ttm/ttm_pool.c | 13 ++++++++-----
> include/drm/ttm/ttm_pool.h | 4 +++-
> 3 files changed, 12 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> index e7147e304637..4a8164a5320f 100644
> --- a/drivers/gpu/drm/ttm/ttm_device.c
> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> @@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
> bdev->funcs = funcs;
>
> ttm_sys_man_init(bdev);
> - ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
> + ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
>
> bdev->vma_manager = vma_manager;
> INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
> index 9f6764bf3b15..1068a41cded1 100644
> --- a/drivers/gpu/drm/ttm/ttm_pool.c
> +++ b/drivers/gpu/drm/ttm/ttm_pool.c
> @@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
> __GFP_KSWAPD_RECLAIM;
>
> if (!pool->use_dma_alloc) {
> - p = alloc_pages(gfp_flags, order);
> + p = alloc_pages_node(pool->nid, gfp_flags, order);
> if (p)
> p->private = order;
> return p;
> @@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
> enum ttm_caching caching,
> unsigned int order)
> {
> - if (pool->use_dma_alloc)
> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
> return &pool->caching[caching].orders[order];
>
> #ifdef CONFIG_X86
> @@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
> *
> * @pool: the pool to initialize
> * @dev: device for DMA allocations and mappings
> + * @nid: NUMA node to use for allocations
> * @use_dma_alloc: true if coherent DMA alloc should be used
> * @use_dma32: true if GFP_DMA32 should be used
> *
> * Initialize the pool and its pool types.
> */
> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> - bool use_dma_alloc, bool use_dma32)
> + int nid, bool use_dma_alloc, bool use_dma32)
> {
> unsigned int i, j;
>
> WARN_ON(!dev && use_dma_alloc);
>
> pool->dev = dev;
> + pool->nid = nid;
> pool->use_dma_alloc = use_dma_alloc;
> pool->use_dma32 = use_dma32;
>
> - if (use_dma_alloc) {
> + if (use_dma_alloc || nid != NUMA_NO_NODE) {
> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> for (j = 0; j < MAX_ORDER; ++j)
> ttm_pool_type_init(&pool->caching[i].orders[j],
> pool, i, j);
> }
> }
> +EXPORT_SYMBOL(ttm_pool_init);
>
> /**
> * ttm_pool_fini - Cleanup a pool
> @@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
> {
> unsigned int i, j;
>
> - if (pool->use_dma_alloc) {
> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> for (j = 0; j < MAX_ORDER; ++j)
> ttm_pool_type_fini(&pool->caching[i].orders[j]);
> diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
> index ef09b23d29e3..23bd8be6d4f8 100644
> --- a/include/drm/ttm/ttm_pool.h
> +++ b/include/drm/ttm/ttm_pool.h
> @@ -61,12 +61,14 @@ struct ttm_pool_type {
> * struct ttm_pool - Pool for all caching and orders
> *
> * @dev: the device we allocate pages for
> + * @nid: which numa node to use
> * @use_dma_alloc: if coherent DMA allocations should be used
> * @use_dma32: if GFP_DMA32 should be used
> * @caching: pools for each caching/order
> */
> struct ttm_pool {
> struct device *dev;
> + int nid;
>
> bool use_dma_alloc;
> bool use_dma32;
> @@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
> void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
>
> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> - bool use_dma_alloc, bool use_dma32);
> + int nid, bool use_dma_alloc, bool use_dma32);
> void ttm_pool_fini(struct ttm_pool *pool);
>
> int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] drm/ttm: add NUMA node id to the pool
2023-03-31 6:54 ` Christian König
@ 2023-03-31 19:17 ` Alex Deucher
2023-03-31 20:02 ` Felix Kuehling
0 siblings, 1 reply; 5+ messages in thread
From: Alex Deucher @ 2023-03-31 19:17 UTC (permalink / raw)
To: Christian König
Cc: Felix Kuehling, Rajneesh Bhardwaj, dri-devel, amd-gfx,
Alex Deucher, Christian König, Graham Sider
On Fri, Mar 31, 2023 at 2:54 AM Christian König
<ckoenig.leichtzumerken@gmail.com> wrote:
>
> Should I push this to drm-misc-next or do we take it through
> amd-staging-drm-next?
I think either way is fine. We can carry it internally as needed for
testing if you want to commit it to drm-misc-next. I don't think
there are any direct code dependencies, but you or Rajneesh can
correct me if I'm wrong.
Alex
>
> Christian.
>
> Am 30.03.23 um 21:50 schrieb Alex Deucher:
> > From: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
> >
> > This allows backing ttm_tt structure with pages from different NUMA
> > pools.
> >
> > Tested-by: Graham Sider <graham.sider@amd.com>
> > Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
> > Signed-off-by: Christian König <christian.koenig@amd.com>
> > Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
> > Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> > ---
> > drivers/gpu/drm/ttm/ttm_device.c | 2 +-
> > drivers/gpu/drm/ttm/ttm_pool.c | 13 ++++++++-----
> > include/drm/ttm/ttm_pool.h | 4 +++-
> > 3 files changed, 12 insertions(+), 7 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> > index e7147e304637..4a8164a5320f 100644
> > --- a/drivers/gpu/drm/ttm/ttm_device.c
> > +++ b/drivers/gpu/drm/ttm/ttm_device.c
> > @@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
> > bdev->funcs = funcs;
> >
> > ttm_sys_man_init(bdev);
> > - ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
> > + ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
> >
> > bdev->vma_manager = vma_manager;
> > INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
> > diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
> > index 9f6764bf3b15..1068a41cded1 100644
> > --- a/drivers/gpu/drm/ttm/ttm_pool.c
> > +++ b/drivers/gpu/drm/ttm/ttm_pool.c
> > @@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
> > __GFP_KSWAPD_RECLAIM;
> >
> > if (!pool->use_dma_alloc) {
> > - p = alloc_pages(gfp_flags, order);
> > + p = alloc_pages_node(pool->nid, gfp_flags, order);
> > if (p)
> > p->private = order;
> > return p;
> > @@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
> > enum ttm_caching caching,
> > unsigned int order)
> > {
> > - if (pool->use_dma_alloc)
> > + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
> > return &pool->caching[caching].orders[order];
> >
> > #ifdef CONFIG_X86
> > @@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
> > *
> > * @pool: the pool to initialize
> > * @dev: device for DMA allocations and mappings
> > + * @nid: NUMA node to use for allocations
> > * @use_dma_alloc: true if coherent DMA alloc should be used
> > * @use_dma32: true if GFP_DMA32 should be used
> > *
> > * Initialize the pool and its pool types.
> > */
> > void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> > - bool use_dma_alloc, bool use_dma32)
> > + int nid, bool use_dma_alloc, bool use_dma32)
> > {
> > unsigned int i, j;
> >
> > WARN_ON(!dev && use_dma_alloc);
> >
> > pool->dev = dev;
> > + pool->nid = nid;
> > pool->use_dma_alloc = use_dma_alloc;
> > pool->use_dma32 = use_dma32;
> >
> > - if (use_dma_alloc) {
> > + if (use_dma_alloc || nid != NUMA_NO_NODE) {
> > for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> > for (j = 0; j < MAX_ORDER; ++j)
> > ttm_pool_type_init(&pool->caching[i].orders[j],
> > pool, i, j);
> > }
> > }
> > +EXPORT_SYMBOL(ttm_pool_init);
> >
> > /**
> > * ttm_pool_fini - Cleanup a pool
> > @@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
> > {
> > unsigned int i, j;
> >
> > - if (pool->use_dma_alloc) {
> > + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
> > for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> > for (j = 0; j < MAX_ORDER; ++j)
> > ttm_pool_type_fini(&pool->caching[i].orders[j]);
> > diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
> > index ef09b23d29e3..23bd8be6d4f8 100644
> > --- a/include/drm/ttm/ttm_pool.h
> > +++ b/include/drm/ttm/ttm_pool.h
> > @@ -61,12 +61,14 @@ struct ttm_pool_type {
> > * struct ttm_pool - Pool for all caching and orders
> > *
> > * @dev: the device we allocate pages for
> > + * @nid: which numa node to use
> > * @use_dma_alloc: if coherent DMA allocations should be used
> > * @use_dma32: if GFP_DMA32 should be used
> > * @caching: pools for each caching/order
> > */
> > struct ttm_pool {
> > struct device *dev;
> > + int nid;
> >
> > bool use_dma_alloc;
> > bool use_dma32;
> > @@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
> > void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
> >
> > void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> > - bool use_dma_alloc, bool use_dma32);
> > + int nid, bool use_dma_alloc, bool use_dma32);
> > void ttm_pool_fini(struct ttm_pool *pool);
> >
> > int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
>
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] drm/ttm: add NUMA node id to the pool
2023-03-31 19:17 ` Alex Deucher
@ 2023-03-31 20:02 ` Felix Kuehling
2023-03-31 21:13 ` Alex Deucher
0 siblings, 1 reply; 5+ messages in thread
From: Felix Kuehling @ 2023-03-31 20:02 UTC (permalink / raw)
To: Alex Deucher, Christian König
Cc: Rajneesh Bhardwaj, dri-devel, amd-gfx, Alex Deucher,
Christian König, Graham Sider
There is a subsequent patch where amdgpu directly calls ttm_pool_init to
create pools per NUMA node. That will depend on the updated function
signature.
Regards,
Felix
On 2023-03-31 15:17, Alex Deucher wrote:
> On Fri, Mar 31, 2023 at 2:54 AM Christian König
> <ckoenig.leichtzumerken@gmail.com> wrote:
>> Should I push this to drm-misc-next or do we take it through
>> amd-staging-drm-next?
> I think either way is fine. We can carry it internally as needed for
> testing if you want to commit it to drm-misc-next. I don't think
> there are any direct code dependencies, but you or Rajneesh can
> correct me if I'm wrong.
>
> Alex
>
>> Christian.
>>
>> Am 30.03.23 um 21:50 schrieb Alex Deucher:
>>> From: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
>>>
>>> This allows backing ttm_tt structure with pages from different NUMA
>>> pools.
>>>
>>> Tested-by: Graham Sider <graham.sider@amd.com>
>>> Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>> Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
>>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
>>> ---
>>> drivers/gpu/drm/ttm/ttm_device.c | 2 +-
>>> drivers/gpu/drm/ttm/ttm_pool.c | 13 ++++++++-----
>>> include/drm/ttm/ttm_pool.h | 4 +++-
>>> 3 files changed, 12 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
>>> index e7147e304637..4a8164a5320f 100644
>>> --- a/drivers/gpu/drm/ttm/ttm_device.c
>>> +++ b/drivers/gpu/drm/ttm/ttm_device.c
>>> @@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
>>> bdev->funcs = funcs;
>>>
>>> ttm_sys_man_init(bdev);
>>> - ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
>>> + ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
>>>
>>> bdev->vma_manager = vma_manager;
>>> INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
>>> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
>>> index 9f6764bf3b15..1068a41cded1 100644
>>> --- a/drivers/gpu/drm/ttm/ttm_pool.c
>>> +++ b/drivers/gpu/drm/ttm/ttm_pool.c
>>> @@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
>>> __GFP_KSWAPD_RECLAIM;
>>>
>>> if (!pool->use_dma_alloc) {
>>> - p = alloc_pages(gfp_flags, order);
>>> + p = alloc_pages_node(pool->nid, gfp_flags, order);
>>> if (p)
>>> p->private = order;
>>> return p;
>>> @@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
>>> enum ttm_caching caching,
>>> unsigned int order)
>>> {
>>> - if (pool->use_dma_alloc)
>>> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
>>> return &pool->caching[caching].orders[order];
>>>
>>> #ifdef CONFIG_X86
>>> @@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
>>> *
>>> * @pool: the pool to initialize
>>> * @dev: device for DMA allocations and mappings
>>> + * @nid: NUMA node to use for allocations
>>> * @use_dma_alloc: true if coherent DMA alloc should be used
>>> * @use_dma32: true if GFP_DMA32 should be used
>>> *
>>> * Initialize the pool and its pool types.
>>> */
>>> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
>>> - bool use_dma_alloc, bool use_dma32)
>>> + int nid, bool use_dma_alloc, bool use_dma32)
>>> {
>>> unsigned int i, j;
>>>
>>> WARN_ON(!dev && use_dma_alloc);
>>>
>>> pool->dev = dev;
>>> + pool->nid = nid;
>>> pool->use_dma_alloc = use_dma_alloc;
>>> pool->use_dma32 = use_dma32;
>>>
>>> - if (use_dma_alloc) {
>>> + if (use_dma_alloc || nid != NUMA_NO_NODE) {
>>> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
>>> for (j = 0; j < MAX_ORDER; ++j)
>>> ttm_pool_type_init(&pool->caching[i].orders[j],
>>> pool, i, j);
>>> }
>>> }
>>> +EXPORT_SYMBOL(ttm_pool_init);
>>>
>>> /**
>>> * ttm_pool_fini - Cleanup a pool
>>> @@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
>>> {
>>> unsigned int i, j;
>>>
>>> - if (pool->use_dma_alloc) {
>>> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
>>> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
>>> for (j = 0; j < MAX_ORDER; ++j)
>>> ttm_pool_type_fini(&pool->caching[i].orders[j]);
>>> diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
>>> index ef09b23d29e3..23bd8be6d4f8 100644
>>> --- a/include/drm/ttm/ttm_pool.h
>>> +++ b/include/drm/ttm/ttm_pool.h
>>> @@ -61,12 +61,14 @@ struct ttm_pool_type {
>>> * struct ttm_pool - Pool for all caching and orders
>>> *
>>> * @dev: the device we allocate pages for
>>> + * @nid: which numa node to use
>>> * @use_dma_alloc: if coherent DMA allocations should be used
>>> * @use_dma32: if GFP_DMA32 should be used
>>> * @caching: pools for each caching/order
>>> */
>>> struct ttm_pool {
>>> struct device *dev;
>>> + int nid;
>>>
>>> bool use_dma_alloc;
>>> bool use_dma32;
>>> @@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
>>> void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
>>>
>>> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
>>> - bool use_dma_alloc, bool use_dma32);
>>> + int nid, bool use_dma_alloc, bool use_dma32);
>>> void ttm_pool_fini(struct ttm_pool *pool);
>>>
>>> int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] drm/ttm: add NUMA node id to the pool
2023-03-31 20:02 ` Felix Kuehling
@ 2023-03-31 21:13 ` Alex Deucher
0 siblings, 0 replies; 5+ messages in thread
From: Alex Deucher @ 2023-03-31 21:13 UTC (permalink / raw)
To: Felix Kuehling
Cc: Christian König, Rajneesh Bhardwaj, dri-devel, amd-gfx,
Alex Deucher, Christian König, Graham Sider
On Fri, Mar 31, 2023 at 4:02 PM Felix Kuehling <felix.kuehling@amd.com> wrote:
>
> There is a subsequent patch where amdgpu directly calls ttm_pool_init to
> create pools per NUMA node. That will depend on the updated function
> signature.
Then we probably want to take this through amdgpu then.
Alex
>
> Regards,
> Felix
>
>
> On 2023-03-31 15:17, Alex Deucher wrote:
> > On Fri, Mar 31, 2023 at 2:54 AM Christian König
> > <ckoenig.leichtzumerken@gmail.com> wrote:
> >> Should I push this to drm-misc-next or do we take it through
> >> amd-staging-drm-next?
> > I think either way is fine. We can carry it internally as needed for
> > testing if you want to commit it to drm-misc-next. I don't think
> > there are any direct code dependencies, but you or Rajneesh can
> > correct me if I'm wrong.
> >
> > Alex
> >
> >> Christian.
> >>
> >> Am 30.03.23 um 21:50 schrieb Alex Deucher:
> >>> From: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
> >>>
> >>> This allows backing ttm_tt structure with pages from different NUMA
> >>> pools.
> >>>
> >>> Tested-by: Graham Sider <graham.sider@amd.com>
> >>> Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
> >>> Signed-off-by: Christian König <christian.koenig@amd.com>
> >>> Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
> >>> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> >>> ---
> >>> drivers/gpu/drm/ttm/ttm_device.c | 2 +-
> >>> drivers/gpu/drm/ttm/ttm_pool.c | 13 ++++++++-----
> >>> include/drm/ttm/ttm_pool.h | 4 +++-
> >>> 3 files changed, 12 insertions(+), 7 deletions(-)
> >>>
> >>> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
> >>> index e7147e304637..4a8164a5320f 100644
> >>> --- a/drivers/gpu/drm/ttm/ttm_device.c
> >>> +++ b/drivers/gpu/drm/ttm/ttm_device.c
> >>> @@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
> >>> bdev->funcs = funcs;
> >>>
> >>> ttm_sys_man_init(bdev);
> >>> - ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
> >>> + ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
> >>>
> >>> bdev->vma_manager = vma_manager;
> >>> INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
> >>> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
> >>> index 9f6764bf3b15..1068a41cded1 100644
> >>> --- a/drivers/gpu/drm/ttm/ttm_pool.c
> >>> +++ b/drivers/gpu/drm/ttm/ttm_pool.c
> >>> @@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
> >>> __GFP_KSWAPD_RECLAIM;
> >>>
> >>> if (!pool->use_dma_alloc) {
> >>> - p = alloc_pages(gfp_flags, order);
> >>> + p = alloc_pages_node(pool->nid, gfp_flags, order);
> >>> if (p)
> >>> p->private = order;
> >>> return p;
> >>> @@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
> >>> enum ttm_caching caching,
> >>> unsigned int order)
> >>> {
> >>> - if (pool->use_dma_alloc)
> >>> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
> >>> return &pool->caching[caching].orders[order];
> >>>
> >>> #ifdef CONFIG_X86
> >>> @@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
> >>> *
> >>> * @pool: the pool to initialize
> >>> * @dev: device for DMA allocations and mappings
> >>> + * @nid: NUMA node to use for allocations
> >>> * @use_dma_alloc: true if coherent DMA alloc should be used
> >>> * @use_dma32: true if GFP_DMA32 should be used
> >>> *
> >>> * Initialize the pool and its pool types.
> >>> */
> >>> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> >>> - bool use_dma_alloc, bool use_dma32)
> >>> + int nid, bool use_dma_alloc, bool use_dma32)
> >>> {
> >>> unsigned int i, j;
> >>>
> >>> WARN_ON(!dev && use_dma_alloc);
> >>>
> >>> pool->dev = dev;
> >>> + pool->nid = nid;
> >>> pool->use_dma_alloc = use_dma_alloc;
> >>> pool->use_dma32 = use_dma32;
> >>>
> >>> - if (use_dma_alloc) {
> >>> + if (use_dma_alloc || nid != NUMA_NO_NODE) {
> >>> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> >>> for (j = 0; j < MAX_ORDER; ++j)
> >>> ttm_pool_type_init(&pool->caching[i].orders[j],
> >>> pool, i, j);
> >>> }
> >>> }
> >>> +EXPORT_SYMBOL(ttm_pool_init);
> >>>
> >>> /**
> >>> * ttm_pool_fini - Cleanup a pool
> >>> @@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
> >>> {
> >>> unsigned int i, j;
> >>>
> >>> - if (pool->use_dma_alloc) {
> >>> + if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
> >>> for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
> >>> for (j = 0; j < MAX_ORDER; ++j)
> >>> ttm_pool_type_fini(&pool->caching[i].orders[j]);
> >>> diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
> >>> index ef09b23d29e3..23bd8be6d4f8 100644
> >>> --- a/include/drm/ttm/ttm_pool.h
> >>> +++ b/include/drm/ttm/ttm_pool.h
> >>> @@ -61,12 +61,14 @@ struct ttm_pool_type {
> >>> * struct ttm_pool - Pool for all caching and orders
> >>> *
> >>> * @dev: the device we allocate pages for
> >>> + * @nid: which numa node to use
> >>> * @use_dma_alloc: if coherent DMA allocations should be used
> >>> * @use_dma32: if GFP_DMA32 should be used
> >>> * @caching: pools for each caching/order
> >>> */
> >>> struct ttm_pool {
> >>> struct device *dev;
> >>> + int nid;
> >>>
> >>> bool use_dma_alloc;
> >>> bool use_dma32;
> >>> @@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
> >>> void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
> >>>
> >>> void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
> >>> - bool use_dma_alloc, bool use_dma32);
> >>> + int nid, bool use_dma_alloc, bool use_dma32);
> >>> void ttm_pool_fini(struct ttm_pool *pool);
> >>>
> >>> int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2023-03-31 21:14 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-03-30 19:50 [PATCH] drm/ttm: add NUMA node id to the pool Alex Deucher
2023-03-31 6:54 ` Christian König
2023-03-31 19:17 ` Alex Deucher
2023-03-31 20:02 ` Felix Kuehling
2023-03-31 21:13 ` Alex Deucher
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.