[PATCH] fs/proc/kcore.c: add mmap interface

[PATCH] fs/proc/kcore.c: add mmap interface

[Feng zhou]

From: ZHOUFENG <zhoufeng.zf@bytedance.com>

When we do the kernel monitor, use the DRGN
(https://github.com/osandov/drgn) access to kernel data structures,
found that the system calls a lot. DRGN is implemented by reading
/proc/kcore. After looking at the kcore code, it is found that kcore
does not implement mmap, resulting in frequent context switching
triggered by read. Therefore, we want to add mmap interface to optimize
performance. Since vmalloc and module areas will change with allocation
and release, consistency cannot be guaranteed, so mmap interface only
maps KCORE_TEXT and KCORE_RAM.

The test results:
1. the default version of kcore
real 11.00
user 8.53
sys 3.59

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
99.64  128.578319          12  11168701           pread64
...
------ ----------- ----------- --------- --------- ----------------
100.00  129.042853              11193748       966 total

2. added kcore for the mmap interface
real 6.44
user 7.32
sys 0.24

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
32.94    0.130120          24      5317       315 futex
11.66    0.046077          21      2231         1 lstat
 9.23    0.036449         177       206           mmap
...
------ ----------- ----------- --------- --------- ----------------
100.00    0.395077                 25435       971 total

The test results show that the number of system calls and time
consumption are significantly reduced.

Co-developed-by: ZHOUFENG Co-Author <zhoufeng.zf@bytedance.com>
Signed-off-by: ZHOUFENG Co-Author <zhoufeng.zf@bytedance.com>
Co-developed-by: CHENYING Co-Author <chenying.kernel@bytedance.com>
Signed-off-by: CHENYING Co-Author <chenying.kernel@bytedance.com>
Signed-off-by: ZHOUFENG <zhoufeng.zf@bytedance.com>
---
 fs/proc/kcore.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/fs/proc/kcore.c b/fs/proc/kcore.c
index 4d2e64e9016c..25a7a9ba2c4a 100644
--- a/fs/proc/kcore.c
+++ b/fs/proc/kcore.c
@@ -573,11 +573,81 @@ static int release_kcore(struct inode *inode, struct file *file)
 	return 0;
 }
 
+static vm_fault_t mmap_kcore_fault(struct vm_fault *vmf)
+{
+	return VM_FAULT_SIGBUS;
+}
+
+static const struct vm_operations_struct kcore_mmap_ops = {
+	.fault = mmap_kcore_fault,
+};
+
+static int mmap_kcore(struct file *file, struct vm_area_struct *vma)
+{
+	size_t size = vma->vm_end - vma->vm_start;
+	u64 start, pfn;
+	int nphdr;
+	size_t data_offset;
+	size_t phdrs_len, notes_len;
+	struct kcore_list *m = NULL;
+	int ret = 0;
+
+	down_read(&kclist_lock);
+
+	get_kcore_size(&nphdr, &phdrs_len, &notes_len, &data_offset);
+
+	start = kc_offset_to_vaddr(((u64)vma->vm_pgoff << PAGE_SHIFT) -
+		((data_offset >> PAGE_SHIFT) << PAGE_SHIFT));
+
+	list_for_each_entry(m, &kclist_head, list) {
+		if (start >= m->addr && size <= m->size)
+			break;
+	}
+
+	if (&m->list == &kclist_head) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (vma->vm_flags & (VM_WRITE | VM_EXEC)) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
+	vma->vm_flags |= VM_MIXEDMAP;
+	vma->vm_ops = &kcore_mmap_ops;
+
+	if (kern_addr_valid(start)) {
+		if (m->type == KCORE_RAM || m->type == KCORE_REMAP)
+			pfn = __pa(start) >> PAGE_SHIFT;
+		else if (m->type == KCORE_TEXT)
+			pfn = __pa_symbol(start) >> PAGE_SHIFT;
+		else {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		if (remap_pfn_range(vma, vma->vm_start, pfn, size,
+				vma->vm_page_prot)) {
+			ret = -EAGAIN;
+			goto out;
+		}
+	} else {
+		ret = -EFAULT;
+	}
+
+out:
+	up_read(&kclist_lock);
+	return ret;
+}
+
 static const struct proc_ops kcore_proc_ops = {
 	.proc_read	= read_kcore,
 	.proc_open	= open_kcore,
 	.proc_release	= release_kcore,
 	.proc_lseek	= default_llseek,
+	.proc_mmap	= mmap_kcore,
 };
 
 /* just remember that we have to update kcore */
-- 
2.11.0

Re: [PATCH] fs/proc/kcore.c: add mmap interface

[Andrew Morton]

On Wed, 26 May 2021 15:51:42 +0800 Feng zhou <zhoufeng.zf@bytedance.com> wrote:

> From: ZHOUFENG <zhoufeng.zf@bytedance.com>
> 
> When we do the kernel monitor, use the DRGN
> (https://github.com/osandov/drgn) access to kernel data structures,
> found that the system calls a lot. DRGN is implemented by reading
> /proc/kcore. After looking at the kcore code, it is found that kcore
> does not implement mmap, resulting in frequent context switching
> triggered by read. Therefore, we want to add mmap interface to optimize
> performance. Since vmalloc and module areas will change with allocation
> and release, consistency cannot be guaranteed, so mmap interface only
> maps KCORE_TEXT and KCORE_RAM.
> 
> The test results:
> 1. the default version of kcore
> real 11.00
> user 8.53
> sys 3.59
> 
> % time     seconds  usecs/call     calls    errors syscall
> ------ ----------- ----------- --------- --------- ----------------
> 99.64  128.578319          12  11168701           pread64
> ...
> ------ ----------- ----------- --------- --------- ----------------
> 100.00  129.042853              11193748       966 total
> 
> 2. added kcore for the mmap interface
> real 6.44
> user 7.32
> sys 0.24
> 
> % time     seconds  usecs/call     calls    errors syscall
> ------ ----------- ----------- --------- --------- ----------------
> 32.94    0.130120          24      5317       315 futex
> 11.66    0.046077          21      2231         1 lstat
>  9.23    0.036449         177       206           mmap
> ...
> ------ ----------- ----------- --------- --------- ----------------
> 100.00    0.395077                 25435       971 total
> 
> The test results show that the number of system calls and time
> consumption are significantly reduced.
> 

hm, OK, I guess why not.  The performance improvements for DRGN (which
appears to be useful) are nice and the code is simple.

I'm surprised that it makes this much difference.  Has DRGN been fully
optimised to minimise the amount of pread()ing which it does?  Why does
it do so much reading?

Thanks, I shall await input from others before moving ahead with this.

Re: [External] Re: [PATCH] fs/proc/kcore.c: add mmap interface

[zhoufeng]

在 2021/5/27 上午8:39, Andrew Morton 写道:
> On Wed, 26 May 2021 15:51:42 +0800 Feng zhou <zhoufeng.zf@bytedance.com> wrote:
> 
>> From: ZHOUFENG <zhoufeng.zf@bytedance.com>
>>
>> When we do the kernel monitor, use the DRGN
>> (https://github.com/osandov/drgn) access to kernel data structures,
>> found that the system calls a lot. DRGN is implemented by reading
>> /proc/kcore. After looking at the kcore code, it is found that kcore
>> does not implement mmap, resulting in frequent context switching
>> triggered by read. Therefore, we want to add mmap interface to optimize
>> performance. Since vmalloc and module areas will change with allocation
>> and release, consistency cannot be guaranteed, so mmap interface only
>> maps KCORE_TEXT and KCORE_RAM.
>>
>> The test results:
>> 1. the default version of kcore
>> real 11.00
>> user 8.53
>> sys 3.59
>>
>> % time     seconds  usecs/call     calls    errors syscall
>> ------ ----------- ----------- --------- --------- ----------------
>> 99.64  128.578319          12  11168701           pread64
>> ...
>> ------ ----------- ----------- --------- --------- ----------------
>> 100.00  129.042853              11193748       966 total
>>
>> 2. added kcore for the mmap interface
>> real 6.44
>> user 7.32
>> sys 0.24
>>
>> % time     seconds  usecs/call     calls    errors syscall
>> ------ ----------- ----------- --------- --------- ----------------
>> 32.94    0.130120          24      5317       315 futex
>> 11.66    0.046077          21      2231         1 lstat
>>   9.23    0.036449         177       206           mmap
>> ...
>> ------ ----------- ----------- --------- --------- ----------------
>> 100.00    0.395077                 25435       971 total
>>
>> The test results show that the number of system calls and time
>> consumption are significantly reduced.
>>
> 
> hm, OK, I guess why not.  The performance improvements for DRGN (which
> appears to be useful) are nice and the code is simple.
> 
> I'm surprised that it makes this much difference.  Has DRGN been fully
> optimised to minimise the amount of pread()ing which it does?  Why does
> it do so much reading?

DRGN is a tool similar to Crash, but much lighter. It allows users to 
obtain kernel data structures from Python scripts. Based on this, we 
intend to use DRGN for kernel monitoring. So we used some pressure test 
scripts to test the loss of monitoring.
Monitoring is all about getting current real-time data, so every time 
DRGN tries to get kernel data, it needs to read /proc/kcore. In my 
script, I tried to loop 1000 times to obtain the information of all the 
processes in the machine, in order to construct a scene where kernel 
data is frequently read. So, the frequency in the default version of 
kcore, pread is very high. In view of this situation, our optimization 
idea is to reduce the number of context switches as much as possible 
under the scenario of frequent kernel data acquisition, to reduce the 
performance loss to a minimum, and then move the monitoring system to 
the production environment.  After running for a long time in a 
production environment, the number of kernel data reads was added as 
time went on, and the pread number also increased. If users use mmap, 
it's once for all.

Attached is the test script:
#!/usr/bin/env drgn
# Copyright (c) Facebook, Inc. and its affiliates.
# SPDX-License-Identifier: GPL-3.0-or-later

"""A simplified implementation of ps(1) using drgn"""

from drgn.helpers.linux.pid import for_each_task

count = 0
while (count < 1000):
     count = count + 1
     #print("PID        COMM")
     for task in for_each_task(prog):
         pid = task.pid.value_()
         comm = task.comm.string_().decode()
         #print(f"{pid:<10} {comm}")

> 
> Thanks, I shall await input from others before moving ahead with this.
> 

Re: [External] Re: [PATCH] fs/proc/kcore.c: add mmap interface

[Andrew Morton]

On Thu, 27 May 2021 14:13:09 +0800 zhoufeng <zhoufeng.zf@bytedance.com> wrote:

> > I'm surprised that it makes this much difference.  Has DRGN been fully
> > optimised to minimise the amount of pread()ing which it does?  Why does
> > it do so much reading?
> DRGN is a tool similar to Crash, but much lighter. It allows users to 
> obtain kernel data structures from Python scripts. Based on this, we 
> intend to use DRGN for kernel monitoring. So we used some pressure test 
> scripts to test the loss of monitoring.
> Monitoring is all about getting current real-time data, so every time 
> DRGN tries to get kernel data, it needs to read /proc/kcore. In my 
> script, I tried to loop 1000 times to obtain the information of all the 
> processes in the machine, in order to construct a scene where kernel 
> data is frequently read. So, the frequency in the default version of 
> kcore, pread is very high. In view of this situation, our optimization 
> idea is to reduce the number of context switches as much as possible 
> under the scenario of frequent kernel data acquisition, to reduce the 
> performance loss to a minimum, and then move the monitoring system to 
> the production environment.

Why would a pread() cause a context switch?

> After running for a long time in a 
> production environment, the number of kernel data reads was added as 
> time went on, and the pread number also increased. If users use mmap, 
> it's once for all.

Re: [External] Re: [PATCH] fs/proc/kcore.c: add mmap interface

[zhoufeng]

在 2021/5/28 上午6:30, Andrew Morton 写道:
> On Thu, 27 May 2021 14:13:09 +0800 zhoufeng <zhoufeng.zf@bytedance.com> wrote:
> 
>>> I'm surprised that it makes this much difference.  Has DRGN been fully
>>> optimised to minimise the amount of pread()ing which it does?  Why does
>>> it do so much reading?
>> DRGN is a tool similar to Crash, but much lighter. It allows users to
>> obtain kernel data structures from Python scripts. Based on this, we
>> intend to use DRGN for kernel monitoring. So we used some pressure test
>> scripts to test the loss of monitoring.
>> Monitoring is all about getting current real-time data, so every time
>> DRGN tries to get kernel data, it needs to read /proc/kcore. In my
>> script, I tried to loop 1000 times to obtain the information of all the
>> processes in the machine, in order to construct a scene where kernel
>> data is frequently read. So, the frequency in the default version of
>> kcore, pread is very high. In view of this situation, our optimization
>> idea is to reduce the number of context switches as much as possible
>> under the scenario of frequent kernel data acquisition, to reduce the
>> performance loss to a minimum, and then move the monitoring system to
>> the production environment.
> 
> Why would a pread() cause a context switch?
> 

Sorry, my English is poor. I mean trigger the system call.

>> After running for a long time in a
>> production environment, the number of kernel data reads was added as
>> time went on, and the pread number also increased. If users use mmap,
>> it's once for all.

Re: [PATCH] fs/proc/kcore.c: add mmap interface

[Andrew Morton]

On Wed, 26 May 2021 15:51:42 +0800 Feng zhou <zhoufeng.zf@bytedance.com> wrote:

> From: ZHOUFENG <zhoufeng.zf@bytedance.com>
> 
> When we do the kernel monitor, use the DRGN
> (https://github.com/osandov/drgn) access to kernel data structures,
> found that the system calls a lot. DRGN is implemented by reading
> /proc/kcore. After looking at the kcore code, it is found that kcore
> does not implement mmap, resulting in frequent context switching
> triggered by read. Therefore, we want to add mmap interface to optimize
> performance. Since vmalloc and module areas will change with allocation
> and release, consistency cannot be guaranteed, so mmap interface only
> maps KCORE_TEXT and KCORE_RAM.
> 
> ...
>
> --- a/fs/proc/kcore.c
> +++ b/fs/proc/kcore.c
> @@ -573,11 +573,81 @@ static int release_kcore(struct inode *inode, struct file *file)
>  	return 0;
>  }
>  
> +static vm_fault_t mmap_kcore_fault(struct vm_fault *vmf)
> +{
> +	return VM_FAULT_SIGBUS;
> +}
> +
> +static const struct vm_operations_struct kcore_mmap_ops = {
> +	.fault = mmap_kcore_fault,
> +};
> +
> +static int mmap_kcore(struct file *file, struct vm_area_struct *vma)
> +{
> +	size_t size = vma->vm_end - vma->vm_start;
> +	u64 start, pfn;
> +	int nphdr;
> +	size_t data_offset;
> +	size_t phdrs_len, notes_len;
> +	struct kcore_list *m = NULL;
> +	int ret = 0;
> +
> +	down_read(&kclist_lock);
> +
> +	get_kcore_size(&nphdr, &phdrs_len, &notes_len, &data_offset);
> +
> +	start = kc_offset_to_vaddr(((u64)vma->vm_pgoff << PAGE_SHIFT) -
> +		((data_offset >> PAGE_SHIFT) << PAGE_SHIFT));
> +
> +	list_for_each_entry(m, &kclist_head, list) {
> +		if (start >= m->addr && size <= m->size)
> +			break;
> +	}
> +
> +	if (&m->list == &kclist_head) {
> +		ret = -EINVAL;
> +		goto out;
> +	}
> +
> +	if (vma->vm_flags & (VM_WRITE | VM_EXEC)) {
> +		ret = -EPERM;
> +		goto out;
> +	}
> +
> +	vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
> +	vma->vm_flags |= VM_MIXEDMAP;
> +	vma->vm_ops = &kcore_mmap_ops;
> +
> +	if (kern_addr_valid(start)) {
> +		if (m->type == KCORE_RAM || m->type == KCORE_REMAP)
> +			pfn = __pa(start) >> PAGE_SHIFT;
> +		else if (m->type == KCORE_TEXT)
> +			pfn = __pa_symbol(start) >> PAGE_SHIFT;
> +		else {
> +			ret = -EFAULT;
> +			goto out;
> +		}
> +
> +		if (remap_pfn_range(vma, vma->vm_start, pfn, size,
> +				vma->vm_page_prot)) {
> +			ret = -EAGAIN;

EAGAIN seems a strange errno for this case.   The mmap manpage says

       EAGAIN The file has been locked, or too much  memory  has  been  locked
              (see setrlimit(2)).


remap_pfn_range() already returns an errno - why not return whatever
that code was?

> +			goto out;
> +		}
> +	} else {
> +		ret = -EFAULT;
> +	}
> +
> +out:
> +	up_read(&kclist_lock);
> +	return ret;
> +}
> +
>  static const struct proc_ops kcore_proc_ops = {
>  	.proc_read	= read_kcore,
>  	.proc_open	= open_kcore,
>  	.proc_release	= release_kcore,
>  	.proc_lseek	= default_llseek,
> +	.proc_mmap	= mmap_kcore,
>  };
>  
>  /* just remember that we have to update kcore */

Otherwise looks OK to me.  Please update the changelog to reflect the
discussion thus far and send a v2?  

Re: [External] Re: [PATCH] fs/proc/kcore.c: add mmap interface

[zhoufeng]

在 2021/6/1 上午9:23, Andrew Morton 写道:
> On Wed, 26 May 2021 15:51:42 +0800 Feng zhou <zhoufeng.zf@bytedance.com> wrote:
> 
>> From: ZHOUFENG <zhoufeng.zf@bytedance.com>
>>
>> When we do the kernel monitor, use the DRGN
>> (https://github.com/osandov/drgn) access to kernel data structures,
>> found that the system calls a lot. DRGN is implemented by reading
>> /proc/kcore. After looking at the kcore code, it is found that kcore
>> does not implement mmap, resulting in frequent context switching
>> triggered by read. Therefore, we want to add mmap interface to optimize
>> performance. Since vmalloc and module areas will change with allocation
>> and release, consistency cannot be guaranteed, so mmap interface only
>> maps KCORE_TEXT and KCORE_RAM.
>>
>> ...
>>
>> --- a/fs/proc/kcore.c
>> +++ b/fs/proc/kcore.c
>> @@ -573,11 +573,81 @@ static int release_kcore(struct inode *inode, struct file *file)
>>   	return 0;
>>   }
>>   
>> +static vm_fault_t mmap_kcore_fault(struct vm_fault *vmf)
>> +{
>> +	return VM_FAULT_SIGBUS;
>> +}
>> +
>> +static const struct vm_operations_struct kcore_mmap_ops = {
>> +	.fault = mmap_kcore_fault,
>> +};
>> +
>> +static int mmap_kcore(struct file *file, struct vm_area_struct *vma)
>> +{
>> +	size_t size = vma->vm_end - vma->vm_start;
>> +	u64 start, pfn;
>> +	int nphdr;
>> +	size_t data_offset;
>> +	size_t phdrs_len, notes_len;
>> +	struct kcore_list *m = NULL;
>> +	int ret = 0;
>> +
>> +	down_read(&kclist_lock);
>> +
>> +	get_kcore_size(&nphdr, &phdrs_len, &notes_len, &data_offset);
>> +
>> +	start = kc_offset_to_vaddr(((u64)vma->vm_pgoff << PAGE_SHIFT) -
>> +		((data_offset >> PAGE_SHIFT) << PAGE_SHIFT));
>> +
>> +	list_for_each_entry(m, &kclist_head, list) {
>> +		if (start >= m->addr && size <= m->size)
>> +			break;
>> +	}
>> +
>> +	if (&m->list == &kclist_head) {
>> +		ret = -EINVAL;
>> +		goto out;
>> +	}
>> +
>> +	if (vma->vm_flags & (VM_WRITE | VM_EXEC)) {
>> +		ret = -EPERM;
>> +		goto out;
>> +	}
>> +
>> +	vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
>> +	vma->vm_flags |= VM_MIXEDMAP;
>> +	vma->vm_ops = &kcore_mmap_ops;
>> +
>> +	if (kern_addr_valid(start)) {
>> +		if (m->type == KCORE_RAM || m->type == KCORE_REMAP)
>> +			pfn = __pa(start) >> PAGE_SHIFT;
>> +		else if (m->type == KCORE_TEXT)
>> +			pfn = __pa_symbol(start) >> PAGE_SHIFT;
>> +		else {
>> +			ret = -EFAULT;
>> +			goto out;
>> +		}
>> +
>> +		if (remap_pfn_range(vma, vma->vm_start, pfn, size,
>> +				vma->vm_page_prot)) {
>> +			ret = -EAGAIN;
> 
> EAGAIN seems a strange errno for this case.   The mmap manpage says
> 
>         EAGAIN The file has been locked, or too much  memory  has  been  locked
>                (see setrlimit(2)).
> 
> 
> remap_pfn_range() already returns an errno - why not return whatever
> that code was?
> 

   yes, that's a good idea.

>> +			goto out;
>> +		}
>> +	} else {
>> +		ret = -EFAULT;
>> +	}
>> +
>> +out:
>> +	up_read(&kclist_lock);
>> +	return ret;
>> +}
>> +
>>   static const struct proc_ops kcore_proc_ops = {
>>   	.proc_read	= read_kcore,
>>   	.proc_open	= open_kcore,
>>   	.proc_release	= release_kcore,
>>   	.proc_lseek	= default_llseek,
>> +	.proc_mmap	= mmap_kcore,
>>   };
>>   
>>   /* just remember that we have to update kcore */
> 
> Otherwise looks OK to me.  Please update the changelog to reflect the
> discussion thus far and send a v2?
> 
   OK, I am very happy to do so, I will send a v2 in two days.